5

Why is this program so slow? I thought the code was fairly optimized, but it takes significantly long than the find command when use on my root filesystem.

It takes about 4 minutes, as opposed to the find command which takes about 40 seconds.

I tried removing the sorting algorithm, but doesn't speed up the program.

package main

import (
    "fmt"
    "io"
    "io/fs"
    "log"
    "os"
    "sort"
    "sync"

    "github.com/google/fscrypt/filesystem"
    "github.com/sirupsen/logrus"
    "gopkg.in/alecthomas/kingpin.v2"
)

var (
    mountpoint = kingpin.Flag("mount", "The mount to find the largest file usages. Can be a subath of mount").Required().String()
    limit      = kingpin.Flag("limit", "The maximum number of files return to the display").Default("10").Short('l').Int()
)
var device string

type fileDisplay struct {
    Size int64
    Path string
}
type bySize []fileDisplay

func (a bySize) Len() int           { return len(a) }
func (a bySize) Less(i, j int) bool { return a[i].Size < a[j].Size }
func (a bySize) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }

var fileChan = make(chan fileDisplay)
var files []fileDisplay

func main() {
    log.SetOutput(io.Discard)
    kingpin.Version("0.0.1")
    kingpin.Parse()
    //Define limit after parsing
    logrus.SetLevel(logrus.FatalLevel)
    if (*mountpoint)[len(*mountpoint)-1:] != "/" {
        *mountpoint = *mountpoint + "/"
    }
    fmt.Println("Finding the top", *limit, "largest files on filesystem", *mountpoint, "\n================================================")
    mount, err := filesystem.FindMount(*mountpoint)
    if err != nil {
        logrus.Fatal(err)
    }
    device = mount.Device

    entries, err := os.ReadDir(*mountpoint)
    if err != nil {
        logrus.Fatal(err)
    }
    var wg sync.WaitGroup
    getFiles(*mountpoint, entries, &wg)
    go func() {
        defer close(fileChan)
        wg.Wait()
    }()
    var last int64
    for file := range fileChan {
        if file.Size > last {
            files = append(files, file)
        } else {
            files = append([]fileDisplay{file}, files...)
        }
    }
    sort.Sort(bySize(files))
    var shortFiles []fileDisplay
    if len(files) > *limit {
        shortFiles = files[len(files)-*limit:]
    } else {
        shortFiles = files
    }

    for _, file := range shortFiles {
        fmt.Println(file.Path, file.DisplaySizeIEC())
    }

}

func getFiles(start string, entries []fs.DirEntry, wg *sync.WaitGroup) {
    for _, entry := range entries {
        wg.Add(1)
        go handleEntry(start, entry, wg)
    }

}

func handleEntry(start string, entry fs.DirEntry, wg *sync.WaitGroup) {
    defer wg.Done()
    var file fileDisplay
    mount, err := filesystem.FindMount(start + entry.Name())
    if err != nil {
        logrus.Fatalln(err, start+entry.Name())
        return
    }
    if mount.Device == device {
        if entry.Type().IsRegular() {
            fileInfo, err := os.Stat(start + entry.Name())
            if err != nil {
                logrus.Fatalln(err, start+entry.Name())
                return
            }
            file.Path = start + entry.Name()
            file.Size = fileInfo.Size()
            fileChan <- file
        } else if entry.IsDir() {
            entries, err := os.ReadDir(start + entry.Name())
            if err != nil {
                logrus.Fatalln(err, start+entry.Name())
                return
            }
            logrus.Info("Searching ", start+entry.Name())
            getFiles(start+entry.Name()+"/", entries, wg)
        }
    }

}

func (f *fileDisplay) DisplaySizeIEC() string {
    const unit = 1024
    b := f.Size
    if b < unit {
        return fmt.Sprintf("%dB", b)
    }
    div, exp := int64(unit), 0
    for n := b / unit; n >= unit; n /= unit {
        div *= unit
        exp++
    }
    return fmt.Sprintf("%.2f%ciB",
        float64(b)/float64(div), "KMGTPE"[exp])
}

Edit: I tried removing the channel and just appending to the slice. This sped it up, but it's not safe because multiple routines could be accessing it.

6
  • One possible issue is that the program does not buffer output. Try this: Add bw := bufio.NewWriter(os.Stdout) before the output loop. Print to the buffered writer: fmt.Fprintln(bw, file.Path, file.DisplaySizeIEC()). Flush the writer after the loop bw.Flush(). Aug 24, 2022 at 20:58
  • @CeriseLimón Thanks. That wasn't the issue. It seems to be choking when ranging over the channel. I believe a single range over the channel is too slow Aug 24, 2022 at 21:04
  • How does the performance compare if you do it serially (no goroutines)?
    – Ry-
    Aug 24, 2022 at 22:25
  • @Ry- It's pretty much the same. Aug 24, 2022 at 22:26
  • 1
    @NorbertvanNobelen I don't have the timings any more, but I did find that the ranging over the channel was causing the slowness. Routines were staying open until the range got to their element. It also caused a big memory problem Aug 25, 2022 at 0:16

1 Answer 1

2

My final draft involved dropping the channel and using sync.RWMutex to lock the list and a custom append function to append with the lock. This allowed me to drop the channel and use append without risking multiple routines editing the same slice.

I dropped the channel because this was causing routines to stay open until the for loop over the open channel could reach their message. My channek operations were blocking. So the routines caused it to slow to the speed of the for loop iterating over the channel.

You can see the differences here:

package main

import (
    "fmt"
    "io"
    "io/fs"
    "log"
    "os"
    "sort"
    "sync"

    "github.com/google/fscrypt/filesystem"
    "github.com/sirupsen/logrus"
    "gopkg.in/alecthomas/kingpin.v2"
)

var (
    mountpoint = kingpin.Flag("mount", "The mount to find the largest file usages. Can be a subath of mount").Required().String()
    limit      = kingpin.Flag("limit", "The maximum number of files return to the display").Default("10").Short('l').Int()
)
var device string

type fileDisplays struct {
    sync.RWMutex
    Files []fileDisplay
}

var files fileDisplays

type fileDisplay struct {
    Size int64
    Path string
}
type bySize []fileDisplay

func (a bySize) Len() int           { return len(a) }
func (a bySize) Less(i, j int) bool { return a[i].Size < a[j].Size }
func (a bySize) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }

func main() {
    log.SetOutput(io.Discard)
    kingpin.Version("0.0.1")
    kingpin.Parse()
    //Define limit after parsing
    logrus.SetLevel(logrus.FatalLevel)
    if (*mountpoint)[len(*mountpoint)-1:] != "/" {
        *mountpoint = *mountpoint + "/"
    }
    fmt.Println("Finding the top", *limit, "largest files on filesystem", *mountpoint, "\n================================================")
    mount, err := filesystem.FindMount(*mountpoint)
    if err != nil {
        logrus.Fatal(err)
    }
    device = mount.Device

    entries, err := os.ReadDir(*mountpoint)
    if err != nil {
        logrus.Fatal(err)
    }
    var wg sync.WaitGroup
    getFiles(*mountpoint, entries, &wg)
    wg.Wait()
    sort.Sort(bySize(files.Files))
    var shortFiles []fileDisplay
    if len(files.Files) > *limit {
        shortFiles = files.Files[len(files.Files)-*limit:]
    } else {
        shortFiles = files.Files
    }

    for _, file := range shortFiles {
        fmt.Println(file.Path, file.DisplaySizeIEC())
    }

}

func getFiles(start string, entries []fs.DirEntry, wg *sync.WaitGroup) {
    for _, entry := range entries {
        wg.Add(1)
        go handleEntry(start, entry, wg)
    }

}

func handleEntry(start string, entry fs.DirEntry, wg *sync.WaitGroup) {
    defer wg.Done()
    var file fileDisplay
    mount, err := filesystem.FindMount(start + entry.Name())
    if err != nil {
        logrus.Errorln(err, start+entry.Name())
        return
    }
    if mount.Device == device {
        if entry.Type().IsRegular() {
            fileInfo, err := os.Stat(start + entry.Name())
            if err != nil {
                logrus.Errorln(err, start+entry.Name())
                return
            }
            file.Path = start + entry.Name()
            file.Size = fileInfo.Size()
            files.Append(file)
        } else if entry.IsDir() {
            entries, err := os.ReadDir(start + entry.Name())
            if err != nil {
                logrus.Errorln(err, start+entry.Name())
                return
            }
            logrus.Info("Searching ", start+entry.Name())
            getFiles(start+entry.Name()+"/", entries, wg)
        }
    }

}

func (f *fileDisplay) DisplaySizeIEC() string {
    const unit = 1024
    b := f.Size
    if b < unit {
        return fmt.Sprintf("%dB", b)
    }
    div, exp := int64(unit), 0
    for n := b / unit; n >= unit; n /= unit {
        div *= unit
        exp++
    }
    return fmt.Sprintf("%.2f%ciB",
        float64(b)/float64(div), "KMGTPE"[exp])
}

func (fd *fileDisplays) Append(item fileDisplay) {
    fd.Lock()
    defer fd.Unlock()

    fd.Files = append(fd.Files, item)
}

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Not the answer you're looking for? Browse other questions tagged or ask your own question.