cmd/go, testing: go fuzz testing get's stuck somewhere else other than the code to be fuzzed

[personnumber3377]

Go version

go version go1.24.4 linux/amd64

Output of go env in your module/workspace:

AR='ar'
CC='gcc'
CGO_CFLAGS='-O2 -g'
CGO_CPPFLAGS=''
CGO_CXXFLAGS='-O2 -g'
CGO_ENABLED='1'
CGO_FFLAGS='-O2 -g'
CGO_LDFLAGS='-O2 -g'
CXX='g++'
GCCGO='gccgo'
GO111MODULE=''
GOAMD64='v1'
GOARCH='amd64'
GOAUTH='netrc'
GOBIN=''
GOCACHE='/home/sontapaa_jokulainen/.cache/go-build'
GOCACHEPROG=''
GODEBUG=''
GOENV='/home/sontapaa_jokulainen/.config/go/env'
GOEXE=''
GOEXPERIMENT=''
GOFIPS140='off'
GOFLAGS=''
GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=/tmp/go-build1537132889=/tmp/go-build -gno-record-gcc-switches'
GOHOSTARCH='amd64'
GOHOSTOS='linux'
GOINSECURE=''
GOMOD='/home/sontapaa_jokulainen/fuzz/archive_fuzzer/go.mod'
GOMODCACHE='/home/sontapaa_jokulainen/go/pkg/mod'
GONOPROXY=''
GONOSUMDB=''
GOOS='linux'
GOPATH='/home/sontapaa_jokulainen/go'
GOPRIVATE=''
GOPROXY='https://proxy.golang.org,direct'
GOROOT='/usr/local/go'
GOSUMDB='sum.golang.org'
GOTELEMETRY='local'
GOTELEMETRYDIR='/home/sontapaa_jokulainen/.config/go/telemetry'
GOTMPDIR=''
GOTOOLCHAIN='auto'
GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64'
GOVCS=''
GOVERSION='go1.24.4'
GOWORK=''
PKG_CONFIG='pkg-config'

What did you do?

Hi!

I am trying to fuzz this program here:

// archive_test.go
package archivedifffuzz

import (
        "archive/zip"
        "bytes"
        "crypto/sha1"
        "encoding/hex"
        "fmt"
        "io"
        "os"
        "os/exec"
        "path/filepath"
        "strings"
        "testing"
        "runtime"
        "context"
        "time"
        "strconv"
)

type FileEntry struct {
        Name string
        Size int
}

func getLocalUnzipPath() string {
        _, filename, _, ok := runtime.Caller(0)
        if !ok {
         panic("unable to get caller info")
        }
        dir := filepath.Dir(filename)
        return filepath.Join(dir, "unzip")
}

func ExtractWithUnzipAndStatFiles(t *testing.T, zipData []byte) (map[string]int, string, error) {
        tmpDir, err := os.MkdirTemp("", "unzipped-*")
        if err != nil {
         return nil, "", err
        }

        tmpZip, err := os.CreateTemp("", "fuzz-*.zip")
        if err != nil {
         os.RemoveAll(tmpDir)
         return nil, "", err
        }
        defer os.Remove(tmpZip.Name())
        defer tmpZip.Close()

        if _, err := tmpZip.Write(zipData); err != nil {
         os.RemoveAll(tmpDir)
         return nil, "", err
        }

        cmd := exec.Command(getLocalUnzipPath(), "-o", tmpZip.Name(), "-d", tmpDir)


        var outBuf bytes.Buffer
        cmd.Stdout = &outBuf
        cmd.Stderr = &outBuf

        if err := cmd.Run(); err != nil {
         os.RemoveAll(tmpDir)
         return nil, "", fmt.Errorf("unzip failed!!!")
        }

        output := outBuf.String()
        if strings.Contains(output, "mismatch") {
         os.RemoveAll(tmpDir)
         return nil, "", fmt.Errorf("filename mismatch detected")
        }

        fileMap := make(map[string]int)
        err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
         if err != nil {
          return err
         }
         if info.IsDir() {
          return nil
         }
         rel, err := filepath.Rel(tmpDir, path)
         if err != nil {
          return err
         }
         fileMap[rel] = int(info.Size())
         return nil
        })
        if err != nil {
         os.RemoveAll(tmpDir)
         return nil, "", err
        }

        
        return fileMap, tmpDir, nil
}
func LoadCorpus(f *testing.F) {
        files, _ := os.ReadDir("corpus/")
        for _, file := range files {
         if data, err := os.ReadFile("corpus/" + file.Name()); err == nil {
          f.Add(data)
         }
        }
}

func FuzzZipDifferential(f *testing.F) {
        f.Add([]byte("PK\x03\x04..."))
        LoadCorpus(f)

        f.Fuzz(func(t *testing.T, data []byte) {
         if len(data) > 100_000 {
          return
         }

         // runDifferentialTest(t, data)


         ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
         defer cancel()

         done := make(chan struct{})
         go func() {
          defer func() {

                if r := recover(); r != nil {
                  panic(r)
                }

                return
          }()

          runDifferentialTest(t, data)
          close(done)
         }()

         select {
         case <-done:
         case <-ctx.Done():
          return
          // panic("timeout")
         }




        })
}

func mapsEqual(a, b map[string]int) bool {
        if len(a) != len(b) {
         return false
        }
        for k, v := range a {
         if b[k] != v {
          return false
         }
        }
        return true
}

func isValidFilename(s string) bool {
        for _, r := range s {
         if r < 32 || r > 127 || r == '\n' || r == '\r' {
          return false
         }
        }
        return true
}

func findFileInZip(reader *zip.Reader, name string) *zip.File {
        for _, f := range reader.File {
         if f.Name == name {
          return f
         }
        }
        return nil
}

func IncrementCounterInFile(filename string) error {
        // Read existing file contents (if any)
        data, err := os.ReadFile(filename)
        count := 0
        if err == nil {
         trimmed := strings.TrimSpace(string(data))
         if trimmed != "" {
          count, err = strconv.Atoi(trimmed)
          if err != nil {
                return fmt.Errorf("invalid number in file: %v", err)
          }
         }
        } else if !os.IsNotExist(err) {
         return err
        }

        // Increment and write back
        count++
        return os.WriteFile(filename, []byte(fmt.Sprintf("%d\n", count)), 0644)
}

func runDifferentialTest(t *testing.T, data []byte) {

        /*
        unzipFiles, unzipDir, err := ExtractWithUnzipAndStatFiles(t, data)
        if err != nil || len(unzipFiles) == 0 {
         return
        }
        defer os.RemoveAll(unzipDir)
        */

        reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
        if err != nil {
         return
        }

        goFiles := make(map[string]int)
        for _, f := range reader.File {
         if !isValidFilename(f.Name) {
          return
         }
         rc, err := f.Open()
         if err != nil {
          return
         }
         n, err := io.Copy(io.Discard, rc)
         rc.Close()
         if err != nil {
          return
         }
         goFiles[f.Name] = int(n)
        }

        // IncrementCounterInFile("/home/YOURHOMEDIRECTORY/integer.txt")
        unzipFiles, unzipDir, err := ExtractWithUnzipAndStatFiles(t, data)
        if err != nil || len(unzipFiles) == 0 {
                // defer os.RemoveAll(unzipDir)
                // t.Errorf("Here is the error: %s", err.Error())
                // panic("fuck")
         return
        }
        // defer os.RemoveAll(unzipDir)

        common := 0
        for name, unzipSize := range unzipFiles {
         if goSize, ok := goFiles[name]; ok {
          common++
          if goSize != unzipSize {
                sum := sha1.Sum(data)
                t.Errorf("Size mismatch for file %s (Go: %d, Unzip: %d)", name, goSize, unzipSize)
                t.Errorf("SHA1: %x", sum)
                if entry := findFileInZip(reader, name); entry != nil {
                  rc, err := entry.Open()
                  if err == nil {
                   goData, _ := io.ReadAll(rc)
                   rc.Close()
                   t.Logf("=== Go version of %s ===\n%s", name, hex.Dump(goData))
                  }
                }
                unzipPath := filepath.Join(unzipDir, name)
                unzipData, err := os.ReadFile(unzipPath)
                if err == nil {
                  t.Logf("=== unzip version of %s ===\n%s", name, hex.Dump(unzipData))
                } else {
                  t.Logf("Failed to read %s from unzip dir: %v", name, err)
                }
                panic("size mismatch")
          }
         }
        }
        // panic("fuck")
        if !mapsEqual(unzipFiles, goFiles) {
         sum := sha1.Sum(data)
         t.Errorf("File mismatch (common: %d, Go: %d, Unzip: %d)", common, len(goFiles), len(unzipFiles))
         t.Errorf("SHA1: %x", sum)
         goOnly := []string{}
         unzipOnly := []string{}
         for name := range goFiles {
          if _, ok := unzipFiles[name]; !ok {
                goOnly = append(goOnly, name)
          }
         }
         for name := range unzipFiles {
          if _, ok := goFiles[name]; !ok {
                unzipOnly = append(unzipOnly, name)
          }
         }
         if len(goOnly) > 0 {
          t.Errorf("Files only in Go archive:\n%s", strings.Join(goOnly, "\n"))
         }
         if len(unzipOnly) > 0 {
          t.Errorf("Files only in Unzip archive:\n%s", strings.Join(unzipOnly, "\n"))
         }
         t.Logf("=== RAW FILENAME DUMP (Go) ===")
         for name := range goFiles {
          t.Logf("%q | bytes: % x", name, []byte(name))
         }
         t.Logf("=== RAW FILENAME DUMP (Unzip) ===")
         for name := range unzipFiles {
          t.Logf("%q | bytes: % x", name, []byte(name))
         }
         panic("filename mismatch")
        }
}

I run it with go test -fuzz=FuzzZipDifferential and then after a while I am getting this output here:

fuzz: elapsed: 38m37s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 38m40s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 38m43s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 38m46s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 38m49s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 38m52s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 38m55s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 38m58s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m1s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m4s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m7s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m10s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m13s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m16s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m19s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m22s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m25s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m28s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m31s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m34s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m37s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m40s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m43s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m46s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m49s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m52s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m55s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 39m58s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 40m1s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)
fuzz: elapsed: 40m4s, execs: 1548489 (0/sec), new interesting: 4 (total: 577)

in the output log. It claims that no executions are done, even though I added the timeout to the code. This leads me to believe that execution gets stuck somewhere else other than the fuzz target code. I have verified that I have disk space on my machine. I also should have adequate memory and processing power.

What did you see happen?

Fuzzing grinds to a halt for some odd reason.

What did you expect to see?

Fuzzing continues normally.

[cagedmantis]

I think it would be helpful if we had a smaller reproducer.

[thepudds]

My recollection is that when it reports execs as 0/sec for a while, that can be the result of it doing significant minimization work during that time period.

I don't know if that explains what is happening here, but I think that at least is or was general behavior that people would find surprising with the builtin go test fuzzer.

For this example in particular, I would be curious if it eventually comes back to reporting non-zero execs per sec if you wait longer, maybe much longer.

[personnumber3377]

Hi!

I added the code to increment an integer in a file to debug if it actually executes anything. This code should get ran even if it is just minimizing the corpus. I uncommented it and when it gets stuck, it doesn't increment the integer in the file when it reports 0 execs a second. This implies that it isn't really minimizing either.

I am going to try to make a more minimal example.

Edit: I am also going to try fuzzing with -fuzzminimizetime 0 (see https://go.dev/doc/security/fuzz/) to disable minimization and to verify that the problem is actually somewhere else.

Edit2: I tried to fuzz using go test -fuzzminimizetime 0 -fuzz=FuzzZipDifferential and now it is not hanging anymore. Maybe the code which hangs indefinitely is in the minimization code itself? Again, the integer in the file did not get incremented during that hang, so it should still get incremented if the minimization proceeds normally, but it didn't so therefore maybe there is a hang in the minimization algorithm itself?

Edit3: Maybe add a feature which shows the minimization etc in the logs too and clarifies if it actually is doing something?

[personnumber3377]

Nevermind. I am able to cause the hang still with this code here:

package archivedifffuzz

import (
        "archive/zip"
        "bytes"
        "crypto/sha1"
        "encoding/hex"
        "fmt"
        "io"
        "os"
        "os/exec"
        "path/filepath"
        "strings"
        "testing"
        "runtime"
        "context"
        "time"
)

type FileEntry struct {
        Name string
        Size int
}

func getLocalUnzipPath() string {
        _, filename, _, ok := runtime.Caller(0)
        if !ok {
                panic("unable to get caller info")
        }
        dir := filepath.Dir(filename)
        return filepath.Join(dir, "unzip")
}

func ExtractWithUnzipAndStatFiles(t *testing.T, zipData []byte) (map[string]int, string, error) {
        tmpDir, err := os.MkdirTemp("", "unzipped-*")
        if err != nil {
                return nil, "", err
        }

        tmpZip, err := os.CreateTemp("", "fuzz-*.zip")
        if err != nil {
                os.RemoveAll(tmpDir)
                return nil, "", err
        }
        defer os.Remove(tmpZip.Name())
        defer tmpZip.Close()

        if _, err := tmpZip.Write(zipData); err != nil {
                os.RemoveAll(tmpDir)
                return nil, "", err
        }

        // cmd := exec.Command("/home/oof/shitfuck/qqq/archive_fuzzer/unzip", "-o", tmpZip.Name(), "-d", tmpDir)
        cmd := exec.Command(getLocalUnzipPath(), "-o", tmpZip.Name(), "-d", tmpDir)


        var outBuf bytes.Buffer
        cmd.Stdout = &outBuf
        cmd.Stderr = &outBuf

        if err := cmd.Run(); err != nil {
                os.RemoveAll(tmpDir)
                return nil, "", fmt.Errorf("unzip failed!!!")
        }

        output := outBuf.String()
        if strings.Contains(output, "mismatch") {
                os.RemoveAll(tmpDir)
                return nil, "", fmt.Errorf("filename mismatch detected")
        }

        fileMap := make(map[string]int)
        err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
                if err != nil {
                        return err
                }
                if info.IsDir() {
                        return nil
                }
                rel, err := filepath.Rel(tmpDir, path)
                if err != nil {
                        return err
                }
                t.Logf("rel: %s\n", rel)
                fileMap[rel] = int(info.Size())
                return nil
        })
        if err != nil {
                os.RemoveAll(tmpDir)
                return nil, "", err
        }

        // ✅ Let caller handle tempDir cleanup after processing
        return fileMap, tmpDir, nil
}
func LoadCorpus(f *testing.F) {
        files, _ := os.ReadDir("corpus/")
        for _, file := range files {
                if data, err := os.ReadFile("corpus/" + file.Name()); err == nil {
                        f.Add(data)
                }
        }
}

func FuzzZipDifferential(f *testing.F) {
        f.Add([]byte("PK\x03\x04..."))
        LoadCorpus(f)

        f.Fuzz(func(t *testing.T, data []byte) {
                if len(data) > 100_000 {
                        return
                }
                //os.WriteFile("/home/oof/fuck.zip", data, 0644)
                //runDifferentialTest(t, data)


                ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
                defer cancel()

                done := make(chan struct{})
                go func() {
                        defer func() {

                                if r := recover(); r != nil {
                                        panic(r)
                                }

                                return
                        }()

                        runDifferentialTest(t, data)
                        close(done)
                }()

                select {
                case <-done:
                case <-ctx.Done():
                        return
                        // panic("timeout")
                }





        })
}

func mapsEqual(a, b map[string]int) bool {
        if len(a) != len(b) {
                return false
        }
        for k, v := range a {
                if b[k] != v {
                        return false
                }
        }
        return true
}

func isValidFilename(s string) bool {
        for _, r := range s {
                if r < 32 || r > 127 || r == '\n' || r == '\r' {
                        return false
                }
        }
        return true
}

func findFileInZip(reader *zip.Reader, name string) *zip.File {
        for _, f := range reader.File {
                if f.Name == name {
                        return f
                }
        }
        return nil
}

func runDifferentialTest(t *testing.T, data []byte) {
        /*
        unzipFiles, unzipDir, err := ExtractWithUnzipAndStatFiles(t, data)
        if err != nil || len(unzipFiles) == 0 {
                return
        }
        defer os.RemoveAll(unzipDir)
        */

        reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
        if err != nil {
                return
        }

        goFiles := make(map[string]int)
        for _, f := range reader.File {
                if !isValidFilename(f.Name) {
                        return
                }
                rc, err := f.Open()
                if err != nil {
                        return
                }
                n, err := io.Copy(io.Discard, rc)
                rc.Close()
                if err != nil {
                        return
                }
                goFiles[f.Name] = int(n)
        }


        unzipFiles, unzipDir, err := ExtractWithUnzipAndStatFiles(t, data)
        if err != nil || len(unzipFiles) == 0 {
                // defer os.RemoveAll(unzipDir)
                // t.Errorf("Here is the error: %s", err.Error())
                // panic("fuck")
                return
        }
        // defer os.RemoveAll(unzipDir)
        // panic("p")
        common := 0
        for name, unzipSize := range unzipFiles {
                if goSize, ok := goFiles[name]; ok {
                        common++
                        if goSize != unzipSize {
                                sum := sha1.Sum(data)
                                t.Errorf("Size mismatch for file %s (Go: %d, Unzip: %d)", name, goSize, unzipSize)
                                t.Errorf("SHA1: %x", sum)
                                if entry := findFileInZip(reader, name); entry != nil {
                                        rc, err := entry.Open()
                                        if err == nil {
                                                goData, _ := io.ReadAll(rc)
                                                rc.Close()
                                                t.Logf("=== Go version of %s ===\n%s", name, hex.Dump(goData))
                                        }
                                }
                                unzipPath := filepath.Join(unzipDir, name)
                                unzipData, err := os.ReadFile(unzipPath)
                                if err == nil {
                                        t.Logf("=== unzip version of %s ===\n%s", name, hex.Dump(unzipData))
                                } else {
                                        t.Logf("Failed to read %s from unzip dir: %v", name, err)
                                }
                                panic("size mismatch")
                        }
                }
        }

        // Remove "files" (actually directories) ending in "/"

        for k := range goFiles {
                if strings.HasSuffix(k, "/") {
                        delete(goFiles, k)
                }
        }

        if !mapsEqual(unzipFiles, goFiles) {
                sum := sha1.Sum(data)
                t.Errorf("File mismatch (common: %d, Go: %d, Unzip: %d)", common, len(goFiles), len(unzipFiles))
                t.Errorf("SHA1: %x", sum)
                goOnly := []string{}
                unzipOnly := []string{}
                for name := range goFiles {
                        if _, ok := unzipFiles[name]; !ok {
                                goOnly = append(goOnly, name)
                        }
                }
                for name := range unzipFiles {
                        if _, ok := goFiles[name]; !ok {
                                unzipOnly = append(unzipOnly, name)
                        }
                }
                if len(goOnly) > 0 {
                        t.Errorf("Files only in Go archive:\n%s", strings.Join(goOnly, "\n"))
                }
                if len(unzipOnly) > 0 {
                        t.Errorf("Files only in Unzip archive:\n%s", strings.Join(unzipOnly, "\n"))
                }
                t.Logf("=== RAW FILENAME DUMP (Go) ===")
                for name := range goFiles {
                        t.Logf("%q | bytes: % x", name, []byte(name))
                }
                t.Logf("=== RAW FILENAME DUMP (Unzip) ===")
                for name := range unzipFiles {
                        t.Logf("%q | bytes: % x", name, []byte(name))
                }
                panic("filename mismatch")
        }
}

and even without corpus minimization: go test -fuzzminimizetime 0 -fuzz=FuzzZipDifferential . Therefore the timeout can not be due to minimization. My code itself has a timeout which I have tested myself to work, therefore it must be somewhere else.

Any help?

[gopherbot]

Timed out in state WaitingForInfo. Closing.

(I am just a bot, though. Please speak up if this is a mistake or you have the requested information.)