decode,interp: Make fuzzing work again and cleanup fatal/error code

Makefile

@@ -90,3 +90,10 @@ update-gomod:
 	GOPROXY=direct go get -d github.com/wader/readline@fq
 	GOPROXY=direct go get -d github.com/wader/gojq@fq
 	go mod tidy
+
+# TODO: as decode recovers panic and "repanics" unrecoverable errors this is a bit hacky at the moment
+# fuzz code is not suppose to print to stderr so log to file
+.PHONY: fuzz
+fuzz:
+# in other terminal: tail -f /tmp/repanic
+	REPANIC_LOG=/tmp/repanic gotip test -tags fuzz -v -fuzz=Fuzz ./format/

format/bzip2/bzip2.go

@@ -92,7 +92,7 @@ func gzDecode(d *decode.D, in interface{}) interface{} {
 	uncompressed := &bytes.Buffer{}
 	crc32W := crc32.NewIEEE()
 	if _, err := decode.Copy(d, io.MultiWriter(uncompressed, crc32W), deflateR); err != nil {
-		d.Invalid(err.Error())
+		d.Fatal(err.Error())
 	}
 	// calculatedCRC32 := crc32W.Sum(nil)
 	uncompressedBB := bitio.NewBufferFromBytes(uncompressed.Bytes(), -1)

format/dns/dns.go

@@ -143,7 +143,7 @@ func fieldFormatLabel(d *decode.D, name string) {
 						}
 						jumpCount++
 						if jumpCount > maxJumps {
-							d.Invalid(fmt.Sprintf("label has more than %d jumps", maxJumps))
+							d.Fatal(fmt.Sprintf("label has more than %d jumps", maxJumps))
 						}
 						d.SeekAbs(int64(pointer) * 8)
 					}

format/elf/elf.go

@@ -146,7 +146,7 @@ func elfDecode(d *decode.D, in interface{}) interface{} {
 	case BIG_ENDIAN:
 		d.Endian = decode.BigEndian
 	default:
-		d.Invalid("unknown endian")
+		d.Fatal("unknown endian")
 	}
 
 	// TODO: hex functions?

format/flac/flac_frame.go

@@ -78,7 +78,7 @@ func utf8Uint(d *decode.D) uint64 {
 			n = n<<6 | d.U8()&0x3f
 		}
 	default:
-		d.Invalid("invalid UTF8Uint")
+		d.Error("invalid UTF8Uint")
 	}
 	return n
 }
@@ -164,7 +164,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 			switch sampleRateBits {
 			case 0:
 				if inStreamInfo == nil {
-					d.Invalid("streaminfo required for sample rate")
+					d.Fatal("streaminfo required for sample rate")
 				}
 				return decode.Scalar{Actual: sampleRateBits, Sym: inStreamInfo.SampleRate, Description: "streaminfo"}
 			case 0b0001:
@@ -256,7 +256,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 			return decode.Scalar{Actual: v, Sym: ch, Description: desc}
 		})
 		if channels == 0 {
-			d.Invalid("unknown number of channels")
+			d.Fatal("unknown number of channels")
 		}
 
 		// <3> Sample size in bits:
@@ -274,7 +274,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 			switch sampleSizeBits {
 			case 0b000:
 				if inStreamInfo == nil {
-					d.Invalid("streaminfo required for bit per sample")
+					d.Fatal("streaminfo required for bit per sample")
 				}
 				sampleSize = int(inStreamInfo.BitPerSample)
 				s.Description = "streaminfo"
@@ -394,7 +394,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 
 				subframeSampleSize := sampleSize - wastedBitsK
 				if subframeSampleSize < 0 {
-					d.Invalid(fmt.Sprintf("negative subframeSampleSize %d", subframeSampleSize))
+					d.Fatal(fmt.Sprintf("negative subframeSampleSize %d", subframeSampleSize))
 				}
 				// if channel is side, add en extra sample bit
 				// https://github.com/xiph/flac/blob/37e675b777d4e0de53ac9ff69e2aea10d92e729c/src/libFLAC/stream_decoder.c#L2040
@@ -405,7 +405,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 
 				decodeWarmupSamples := func(samples []int64, n int, sampleSize int) {
 					if len(samples) < n {
-						d.Invalid("decodeWarmupSamples outside block size")
+						d.Fatal("decodeWarmupSamples outside block size")
 					}
 
 					d.FieldArray("warmup_samples", func(d *decode.D) {
@@ -473,7 +473,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 								riceParameter := int(d.FieldU("rice_parameter", riceBits))
 
 								if samplesLen < n+count {
-									d.Invalid("decodeResiduals outside block size")
+									d.Fatal("decodeResiduals outside block size")
 								}
 
 								if riceParameter == riceEscape {
@@ -559,7 +559,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 					// <5> Quantized linear predictor coefficient shift needed in bits (NOTE: this number is signed two's-complement).
 					shift := d.FieldS5("shift")
 					if shift < 0 {
-						d.Invalid(fmt.Sprintf("negative LPC shift %d", shift))
+						d.Fatal(fmt.Sprintf("negative LPC shift %d", shift))
 					}
 					// <n> Unencoded predictor coefficients (n = qlp coeff precision * lpc order) (NOTE: the coefficients are signed two's-complement).
 					var coeffs []int64
@@ -594,7 +594,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 	streamSamples := len(channelSamples[0])
 	for j := 0; j < len(channelSamples); j++ {
 		if streamSamples > len(channelSamples[j]) {
-			d.Invalid(fmt.Sprintf("different amount of samples in channels %d != %d", streamSamples, len(channelSamples[j])))
+			d.Fatal(fmt.Sprintf("different amount of samples in channels %d != %d", streamSamples, len(channelSamples[j])))
 		}
 	}
 

format/flac/flac_metadatablock.go

@@ -60,7 +60,7 @@ func metadatablockDecode(d *decode.D, in interface{}) interface{} {
 	case MetadataBlockStreaminfo:
 		flacStreaminfoOut, ok := d.Format(flacStreaminfoFormat, nil).(format.FlacStreaminfoOut)
 		if !ok {
-			d.Invalid(fmt.Sprintf("expected FlacStreaminfoOut, got %#+v", flacStreaminfoOut))
+			panic(fmt.Sprintf("expected FlacStreaminfoOut, got %#+v", flacStreaminfoOut))
 		}
 		hasStreamInfo = true
 		streamInfo = flacStreaminfoOut.StreamInfo

format/flac/flac_metadatablocks.go

@@ -28,10 +28,10 @@ func metadatablocksDecode(d *decode.D, in interface{}) interface{} {
 
 	isLastBlock := false
 	for !isLastBlock {
-		_, flacMetadatablockOutAny := d.FieldFormat("metadatablock", flacMetadatablockForamt, nil)
-		flacMetadatablockOut, ok := flacMetadatablockOutAny.(format.FlacMetadatablockOut)
-		if !ok {
-			d.Invalid(fmt.Sprintf("expected FlacMetadatablocksOut, got %#+v", flacMetadatablockOut))
+		v, dv := d.FieldFormat("metadatablock", flacMetadatablockForamt, nil)
+		flacMetadatablockOut, ok := dv.(format.FlacMetadatablockOut)
+		if v != nil && !ok {
+			panic(fmt.Sprintf("expected FlacMetadatablocksOut, got %#+v", flacMetadatablockOut))
 		}
 		isLastBlock = flacMetadatablockOut.IsLastBlock
 		if flacMetadatablockOut.HasStreamInfo {

format/fuzz_test.go

@@ -1,43 +1,62 @@
-//go:build gofuzzbeta
+//go:build fuzz
 
 package format_test
 
 import (
 	"bytes"
 	"context"
 	"fmt"
-	_ "github.com/wader/fq/format/all"
-	"github.com/wader/fq/pkg/interp"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
+
+	_ "github.com/wader/fq/format/all"
+	"github.com/wader/fq/format/registry"
+	"github.com/wader/fq/pkg/interp"
 )
 
+type fuzzFS struct{}
+
+func (fuzzFS) Open(name string) (fs.File, error) {
+	return nil, fmt.Errorf("%s: file not found", name)
+}
+
 type fuzzTest struct {
 	b []byte
 }
 
+type fuzzTestInput struct {
+	interp.FileReader
+	io.Writer
+}
+
+func (fuzzTestInput) IsTerminal() bool { return false }
+func (fuzzTestInput) Size() (int, int) { return 120, 25 }
+
 type fuzzTestOutput struct {
 	io.Writer
 }
 
 func (o fuzzTestOutput) Size() (int, int) { return 120, 25 }
 func (o fuzzTestOutput) IsTerminal() bool { return false }
 
-func (ft *fuzzTest) Stdin() io.Reader             { return bytes.NewBuffer(ft.b) } // TODO: special file?
+func (ft *fuzzTest) Stdin() interp.Input {
+	return fuzzTestInput{FileReader: interp.FileReader{R: bytes.NewBuffer(ft.b)}}
+}
 func (ft *fuzzTest) Stdout() interp.Output        { return fuzzTestOutput{os.Stdout} }
-func (ft *fuzzTest) Stderr() io.Writer            { return os.Stderr }
+func (ft *fuzzTest) Stderr() interp.Output        { return fuzzTestOutput{os.Stderr} }
 func (ft *fuzzTest) InterruptChan() chan struct{} { return nil }
 func (ft *fuzzTest) Environ() []string            { return nil }
 func (ft *fuzzTest) Args() []string {
 	return []string{}
 }
 func (ft *fuzzTest) ConfigDir() (string, error) { return "/config", nil }
-func (ft *fuzzTest) Open(name string) (io.ReadSeeker, error) {
-	return nil, fmt.Errorf("%s: file not found", name)
-}
+func (ft *fuzzTest) FS() fs.FS                  { return fuzzFS{} }
+func (ft *fuzzTest) History() ([]string, error) { return nil, nil }
+
 func (ft *fuzzTest) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
 	return "", io.EOF
 }
@@ -67,15 +86,15 @@ func FuzzFormats(f *testing.F) {
 
 	f.Fuzz(func(t *testing.T, b []byte) {
 		fz := &fuzzTest{b: b}
-		q, err := interp.New(fz, all.Registry)
+		q, err := interp.New(fz, registry.Default)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		err = q.Main(context.Background(), fz.Stdout(), "dev")
-		if err != nil {
-			// TODO: expect error
-			t.Fatal(err)
-		}
+		_ = q.Main(context.Background(), fz.Stdout(), "dev")
+		// if err != nil {
+		// 	// TODO: expect error
+		// 	t.Fatal(err)
+		// }
 	})
 }

format/gif/gif.go

@@ -142,6 +142,8 @@ func gifDecode(d *decode.D, in interface{}) interface{} {
 						}
 					})
 				})
+			default:
+				d.Fatal("unknown block")
 			}
 		}
 	})

format/gzip/gzip.go

@@ -106,7 +106,7 @@ func gzDecode(d *decode.D, in interface{}) interface{} {
 		deflateR := flate.NewReader(compressedBB)
 		uncompressed := &bytes.Buffer{}
 		if _, err := decode.Copy(d, io.MultiWriter(uncompressed, crc32W), deflateR); err != nil {
-			d.Invalid(err.Error())
+			d.Fatal(err.Error())
 		}
 		uncompressedBB := bitio.NewBufferFromBytes(uncompressed.Bytes(), -1)
 		dv, _, _ := d.FieldTryFormatBitBuf("uncompressed", uncompressedBB, probeFormat, nil)

format/id3/id3v1.go

@@ -21,7 +21,7 @@ func id3v1Decode(d *decode.D, in interface{}) interface{} {
 	d.AssertAtLeastBitsLeft(128 * 8)
 	d.FieldUTF8("magic", 3, d.AssertStr("TAG"))
 	if d.PeekBits(8) == uint64('+') {
-		d.Invalid("looks like id3v11")
+		d.Error("looks like id3v11")
 	}
 	d.FieldUTF8NullTerminatedLen("song_name", 30)
 	d.FieldUTF8NullTerminatedLen("artist", 30)

format/id3/id3v2.go

@@ -382,6 +382,9 @@ func decodeFrame(d *decode.D, version int) uint64 {
 		}
 
 		size = dataSize + headerLen
+	default:
+		// can't know size
+		d.Fatal("unknown version")
 	}
 
 	// note frame function run inside a SubLenFn so they can use BitLefts and
@@ -523,7 +526,7 @@ func decodeFrame(d *decode.D, version int) uint64 {
 		idNormalized = "COMM"
 	case id == "TXX", id == "TXXX":
 		idNormalized = "TXXX"
-	case id[0] == 'T':
+	case len(id) > 0 && id[0] == 'T':
 		idNormalized = "T000"
 	}
 
@@ -578,7 +581,7 @@ func id3v2Decode(d *decode.D, in interface{}) interface{} {
 	version := int(d.FieldU8("version"))
 	versionValid := version == 2 || version == 3 || version == 4
 	if !versionValid {
-		d.Invalid(fmt.Sprintf("unsupported version %d", version))
+		d.Fatal(fmt.Sprintf("unsupported version %d", version))
 	}
 
 	d.FieldU8("revision")

format/jpeg/jpeg.go

@@ -167,7 +167,7 @@ var markers = decode.UToScalar{
 func jpegDecode(d *decode.D, in interface{}) interface{} {
 	d.AssertLeastBytesLeft(2)
 	if !bytes.Equal(d.PeekBytes(2), []byte{0xff, SOI}) {
-		d.Invalid("no SOI marker")
+		d.Error("no SOI marker")
 	}
 
 	var extendedXMP []byte
@@ -268,7 +268,7 @@ func jpegDecode(d *decode.D, in interface{}) interface{} {
 						eoiMarkerFound = true
 					default:
 						if !markerFound {
-							d.Invalid(fmt.Sprintf("unknown marker %x", markerCode))
+							d.Error(fmt.Sprintf("unknown marker %x", markerCode))
 						}
 
 						markerLen := d.FieldU16("length")
@@ -308,7 +308,7 @@ func jpegDecode(d *decode.D, in interface{}) interface{} {
 									// TODO: redo this? multi reader?
 									chunkBytes, err := chunk.Bytes()
 									if err != nil {
-										d.Invalid(fmt.Sprintf("failed to read xmp chunk: %s", err))
+										d.Fatal(fmt.Sprintf("failed to read xmp chunk: %s", err))
 									}
 
 									if extendedXMP == nil {
@@ -344,7 +344,7 @@ func jpegDecode(d *decode.D, in interface{}) interface{} {
 	})
 
 	if !soiMarkerFound {
-		d.Invalid("no SOI marker found")
+		d.Error("no SOI marker found")
 	}
 
 	if extendedXMP != nil {

format/json/json.go

@@ -23,13 +23,13 @@ func decodeJSON(d *decode.D, in interface{}) interface{} {
 	jd := stdjson.NewDecoder(bb)
 	var s decode.Scalar
 	if err := jd.Decode(&s.Actual); err != nil {
-		d.Invalid(err.Error())
+		d.Fatal(err.Error())
 	}
 	switch s.Actual.(type) {
 	case map[string]interface{},
 		[]interface{}:
 	default:
-		d.Invalid("root not object or array")
+		d.Fatal("root not object or array")
 	}
 	// TODO: root not array/struct how to add unknown gaps?
 	// TODO: ranges not end up correct