interp,decode: Add force option to ignore asserts

doc/usage.md

@@ -108,7 +108,7 @@ Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
 --argjson NAME JSON      Set variable $NAME to JSON
 --color-output,-C        Force color output
 --compact-output,-c      Compact output
---decode,-d NAME         Force decode format (probe)
+--decode,-d NAME         Decode format (probe)
 --decode-file NAME PATH  Set variable $NAME to decode of file
 --formats                Show supported formats
 --from-file,-f PATH      Read EXPR from file
@@ -153,12 +153,12 @@ notable is support for arbitrary-precision integers.
 - All standard library functions from jq
 - Adds a few new general functions:
   - `streaks/0`, `streaks_by/1` like `group` but groups streaks based on condition.
-  - `count`, `count_by/1` like `group` but counts groups lengths.
+  - `count/0`, `count_by/1` like `group` but counts groups lengths.
   - `debug/1` like `debug/0` but uses arg to produce debug message. `{a: 123} | debug({a}) | ...`.
-  - `path_to_expr` from `["key", 1]` to `".key[1]"`.
-  - `expr_to_path` from `".key[1]"` to `["key", 1]`.
+  - `path_to_expr/0` from `["key", 1]` to `".key[1]"`.
+  - `expr_to_path/0` from `".key[1]"` to `["key", 1]`.
   - `diff/2` produce diff object between two values.
-  - `delta`, `delta_by/1`, array with difference between all consecutive pairs.
+  - `delta/0`, `delta_by/1`, array with difference between all consecutive pairs.
   - `chunk/1`, split array or string into even chunks
 - Adds some decode value specific functions:
   - `root/0` return tree root for value
@@ -181,20 +181,25 @@ notable is support for arbitrary-precision integers.
     - `bgrep/1`, `bgrep/2` recursively match buffer
     - `fgrep/1`, `fgrep/2` recursively match field name
   - Buffers:
-    - `tobits` - Transform input into a bits buffer not preserving source range, will start at zero.
-    - `tobitsrange` - Transform input into a bits buffer preserving source range if possible.
-    - `tobytes` - Transform input into a bytes buffer not preserving source range, will start at zero.
-    - `tobytesrange` - Transform input into a byte buffer preserving source range if possible.
+    - `tobits/0` - Transform input into a bits buffer not preserving source range, will start at zero.
+    - `tobitsrange/0` - Transform input into a bits buffer preserving source range if possible.
+    - `tobytes/0` - Transform input into a bytes buffer not preserving source range, will start at zero.
+    - `tobytesrange/0` - Transform input into a byte buffer preserving source range if possible.
     - `buffer[start:end]`, `buffer[:end]`, `buffer[start:]` - Create a sub buffer from start to end in buffer units preserving source range.
 - `open` open file for reading
-- `probe` or `decode` probe format and decode
-- `mp3`, `matroska`, ..., `<name>`, `decode([name])` force decode as format
-- `d`/`display` display value and truncate long arrays
-- `f`/`full` display value and don't truncate arrays
-- `v`/`verbose` display value verbosely and don't truncate array
-- `p`/`preview` show preview of field tree
-- `hd`/`hexdump` hexdump value
-- `repl` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" multiple outputs `1, 2, 3 | repl`.
+- All decode function takes a optional option argument. The only option currently is `force` to ignore decoder asserts.
+For example to decode as mp3 and ignore assets do `mp3({force: true})` or `decode("mp3"; {force: true})`, from command line
+you currently have to do `fq -d raw 'mp3({force: true})' file`.
+- `decode/0`, `decode/1`, `decode/2` decode format
+- `probe/0`, `probe/1` probe and decode format
+- `mp3/0`, `mp3/1`, ..., `<name>/0`, `<name>/1` same as `decode(<name>)/1`, `decode(<name>; <opts>)/2`  decode as format
+
+- `d/0`/`display/0` display value and truncate long arrays
+- `f/0`/`full/0` display value and don't truncate arrays
+- `v/0`/`verbose/0` display value verbosely and don't truncate array
+- `p/0`/`preview/0` show preview of field tree
+- `hd/0`/`hexdump/0` hexdump value
+- `repl/0` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" multiple outputs `1, 2, 3 | repl`.
 
 ## Decoded values (TODO: better name?)
 

format/elf/elf.go

@@ -131,7 +131,7 @@ func elfDecode(d *decode.D, in interface{}) interface{} {
 	var endian uint64
 
 	d.FieldStruct("ident", func(d *decode.D) {
-		d.FieldRawLen("magic", 4*8, d.AssertRaw([]byte("\x7fELF")))
+		d.FieldRawLen("magic", 4*8, d.AssertBitBuf([]byte("\x7fELF")))
 		archBits = int(d.FieldU8("class", d.MapUToU(classBits)))
 		endian = d.FieldU8("data", d.MapUToStr(endianNames))
 		d.FieldU8("version")

format/flac/flac.go

@@ -78,7 +78,7 @@ func flacDecode(d *decode.D, in interface{}) interface{} {
 		}
 	})
 
-	d.FieldValueRaw("md5_calculated", md5Samples.Sum(nil), d.ValidateRaw(streamInfo.MD5), d.RawHex)
+	d.FieldValueRaw("md5_calculated", md5Samples.Sum(nil), d.ValidateBitBuf(streamInfo.MD5), d.RawHex)
 	d.FieldValueU("decoded_samples", framesNDecodedSamples)
 
 	return nil

format/flac/flac_frame.go

@@ -339,7 +339,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 
 		headerCRC := &crc.CRC{Bits: 8, Table: crc.ATM8Table}
 		decode.MustCopy(d, headerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
-		d.FieldRawLen("crc", 8, d.ValidateRaw(headerCRC.Sum(nil)), d.RawHex)
+		d.FieldRawLen("crc", 8, d.ValidateBitBuf(headerCRC.Sum(nil)), d.RawHex)
 	})
 
 	var channelSamples [][]int64
@@ -589,7 +589,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 	// <16> CRC-16 (polynomial = x^16 + x^15 + x^2 + x^0, initialized with 0) of everything before the crc, back to and including the frame header sync code
 	footerCRC := &crc.CRC{Bits: 16, Table: crc.ANSI16Table}
 	decode.MustCopy(d, footerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
-	d.FieldRawLen("footer_crc", 16, d.ValidateRaw(footerCRC.Sum(nil)), d.RawHex)
+	d.FieldRawLen("footer_crc", 16, d.ValidateBitBuf(footerCRC.Sum(nil)), d.RawHex)
 
 	streamSamples := len(channelSamples[0])
 	for j := 0; j < len(channelSamples); j++ {

format/gzip/gzip.go

@@ -59,7 +59,7 @@ var deflateExtraFlagsNames = decode.UToStr{
 }
 
 func gzDecode(d *decode.D, in interface{}) interface{} {
-	d.FieldRawLen("identification", 2*8, d.AssertRaw([]byte("\x1f\x8b")))
+	d.FieldRawLen("identification", 2*8, d.AssertBitBuf([]byte("\x1f\x8b")))
 	compressionMethod := d.FieldU8("compression_method", d.MapUToStr(compressionMethodNames))
 	hasHeaderCRC := false
 	hasExtra := false
@@ -117,7 +117,7 @@ func gzDecode(d *decode.D, in interface{}) interface{} {
 		d.FieldRawLen("compressed", compressedLen)
 	}
 
-	d.FieldRawLen("crc32", 32, d.ValidateRaw(bitio.ReverseBytes(crc32W.Sum(nil))), d.RawHex)
+	d.FieldRawLen("crc32", 32, d.ValidateBitBuf(bitio.ReverseBytes(crc32W.Sum(nil))), d.RawHex)
 	d.FieldU32LE("isize")
 
 	return nil

format/jpeg/jpeg.go

@@ -192,7 +192,7 @@ func jpegDecode(d *decode.D, in interface{}) interface{} {
 			} else {
 				d.FieldStruct("marker", func(d *decode.D) {
 					prefixLen := d.PeekFindByte(0xff, -1) + 1
-					d.FieldRawLen("prefix", prefixLen*8, d.AssertRaw([]byte{0xff}))
+					d.FieldRawLen("prefix", prefixLen*8, d.AssertBitBuf([]byte{0xff}))
 					markerCode := d.FieldU8("code", d.MapUToScalar(markers))
 					_, markerFound := markers[markerCode]
 

format/mpeg/mp3_frame.go

@@ -388,7 +388,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 	decode.MustCopy(d, crcHash, d.BitBufRange(6*8, sideInfoBytes*8))
 
 	if crcValue != nil {
-		_ = crcValue.ScalarFn(d.ValidateRaw(crcHash.Sum(nil)))
+		_ = crcValue.ScalarFn(d.ValidateBitBuf(crcHash.Sum(nil)))
 	}
 	d.FieldValueRaw("crc_calculated", crcHash.Sum(nil), d.RawHex)
 

format/ogg/ogg_page.go

@@ -51,7 +51,7 @@ func pageDecode(d *decode.D, in interface{}) interface{} {
 	decode.MustCopy(d, pageCRC, d.BitBufRange(startPos, pageChecksumValue.Range.Start-startPos))                      // header before checksum
 	decode.MustCopy(d, pageCRC, bytes.NewReader([]byte{0, 0, 0, 0}))                                                  // zero checksum bits
 	decode.MustCopy(d, pageCRC, d.BitBufRange(pageChecksumValue.Range.Stop(), endPos-pageChecksumValue.Range.Stop())) // rest of page
-	_ = pageChecksumValue.ScalarFn(d.ValidateRaw(bitio.ReverseBytes(pageCRC.Sum(nil))))
+	_ = pageChecksumValue.ScalarFn(d.ValidateBitBuf(bitio.ReverseBytes(pageCRC.Sum(nil))))
 
 	return p
 }

format/png/png.go

@@ -64,7 +64,7 @@ var blendOpNames = decode.UToStr{
 func pngDecode(d *decode.D, in interface{}) interface{} {
 	iEndFound := false
 
-	d.FieldRawLen("signature", 8*8, d.AssertRaw([]byte("\x89PNG\r\n\x1a\n")))
+	d.FieldRawLen("signature", 8*8, d.AssertBitBuf([]byte("\x89PNG\r\n\x1a\n")))
 	d.FieldStructArrayLoop("chunks", "chunk", func() bool { return d.NotEnd() && !iEndFound }, func(d *decode.D) {
 		chunkLength := int(d.FieldU32("length"))
 		crcStartPos := d.Pos()
@@ -192,7 +192,7 @@ func pngDecode(d *decode.D, in interface{}) interface{} {
 
 		chunkCRC := crc32.NewIEEE()
 		decode.MustCopy(d, chunkCRC, d.BitBufRange(crcStartPos, d.Pos()-crcStartPos))
-		d.FieldRawLen("crc", 32, d.ValidateRaw(chunkCRC.Sum(nil)), d.RawHex)
+		d.FieldRawLen("crc", 32, d.ValidateBitBuf(chunkCRC.Sum(nil)), d.RawHex)
 	})
 
 	return nil

pkg/decode/decode.go

@@ -24,6 +24,7 @@ const (
 type Options struct {
 	Name          string
 	Description   string
+	Force         bool
 	FillGaps      bool
 	IsRoot        bool
 	Range         ranges.Range // if zero use whole buffer
@@ -38,20 +39,19 @@ func Decode(ctx context.Context, bb *bitio.Buffer, formats []*Format, opts Optio
 }
 
 func decode(ctx context.Context, bb *bitio.Buffer, formats []*Format, opts Options) (*Value, interface{}, error) {
-	if opts.Range.IsZero() {
-		opts.Range = ranges.Range{Len: bb.Len()}
+	decodeRange := opts.Range
+	if decodeRange.IsZero() {
+		decodeRange = ranges.Range{Len: bb.Len()}
 	}
 
 	if formats == nil {
 		panic("formats is nil, failed to register format?")
 	}
 
-	var forceOne = len(formats) == 1
-
 	decodeErr := FormatsError{}
 
 	for _, f := range formats {
-		cbb, err := bb.BitBufRange(opts.Range.Start, opts.Range.Len)
+		cbb, err := bb.BitBufRange(decodeRange.Start, decodeRange.Len)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -84,7 +84,7 @@ func decode(ctx context.Context, bb *bitio.Buffer, formats []*Format, opts Optio
 					d.Value.V = vv
 				}
 
-				if !forceOne {
+				if len(formats) != 1 {
 					continue
 				}
 			} else {
@@ -94,20 +94,20 @@ func decode(ctx context.Context, bb *bitio.Buffer, formats []*Format, opts Optio
 
 		// TODO: maybe move to Format* funcs?
 		if opts.FillGaps {
-			d.FillGaps(ranges.Range{Start: 0, Len: opts.Range.Len}, "unknown")
+			d.FillGaps(ranges.Range{Start: 0, Len: decodeRange.Len}, "unknown")
 		}
 
 		var minMaxRange ranges.Range
 		if err := d.Value.WalkRootPreOrder(func(v *Value, rootV *Value, depth int, rootDepth int) error {
 			minMaxRange = ranges.MinMax(minMaxRange, v.Range)
-			v.Range.Start += opts.Range.Start
+			v.Range.Start += decodeRange.Start
 			v.RootBitBuf = bb
 			return nil
 		}); err != nil {
 			return nil, nil, err
 		}
 
-		d.Value.Range = ranges.Range{Start: opts.Range.Start, Len: minMaxRange.Len}
+		d.Value.Range = ranges.Range{Start: decodeRange.Start, Len: minMaxRange.Len}
 
 		if opts.IsRoot {
 			d.Value.postProcess()
@@ -123,7 +123,7 @@ type D struct {
 	Ctx     context.Context
 	Endian  Endian
 	Value   *Value
-	Options map[string]interface{}
+	Options Options
 
 	bitBuf *bitio.Buffer
 
@@ -154,7 +154,7 @@ func newDecoder(ctx context.Context, format *Format, bb *bitio.Buffer, opts Opti
 			Range:      ranges.Range{Start: 0, Len: 0},
 			IsRoot:     opts.IsRoot,
 		},
-		Options: opts.FormatOptions,
+		Options: opts,
 
 		bitBuf:  bb,
 		readBuf: opts.ReadBuf,
@@ -217,7 +217,9 @@ func (d *D) FillGaps(r ranges.Range, namePrefix string) {
 
 // Invalid stops decode with a reason
 func (d *D) Invalid(reason string) {
-	panic(ValidateError{Reason: reason, Pos: d.Pos()})
+	if !d.Options.Force {
+		panic(ValidateError{Reason: reason, Pos: d.Pos()})
+	}
 }
 
 func (d *D) IOPanic(err error) {
@@ -556,6 +558,9 @@ func (d *D) FieldRangeFn(name string, firstBit int64, nBits int64, fn func() *Va
 }
 
 func (d *D) AssertAtLeastBitsLeft(nBits int64) {
+	if d.Options.Force {
+		return
+	}
 	bl := d.BitsLeft()
 	if bl < nBits {
 		// TODO:
@@ -564,6 +569,9 @@ func (d *D) AssertAtLeastBitsLeft(nBits int64) {
 }
 
 func (d *D) AssertLeastBytesLeft(nBytes int64) {
+	if d.Options.Force {
+		return
+	}
 	bl := d.BitsLeft()
 	if bl < nBytes*8 {
 		// TODO:
@@ -641,11 +649,12 @@ func (d *D) RangeFn(firstBit int64, nBits int64, fn func(d *D)) {
 
 func (d *D) Format(formats []*Format, inArg interface{}) interface{} {
 	dv, v, err := decode(d.Ctx, d.bitBuf, formats, Options{
-		ReadBuf:     d.readBuf,
+		Force:       d.Options.Force,
 		FillGaps:    false,
 		IsRoot:      false,
 		Range:       ranges.Range{Start: d.Pos(), Len: d.BitsLeft()},
 		FormatInArg: inArg,
+		ReadBuf:     d.readBuf,
 	})
 	if dv == nil || dv.Errors() != nil {
 		panic(err)
@@ -670,11 +679,12 @@ func (d *D) Format(formats []*Format, inArg interface{}) interface{} {
 func (d *D) FieldTryFormat(name string, formats []*Format, inArg interface{}) (*Value, interface{}, error) {
 	dv, v, err := decode(d.Ctx, d.bitBuf, formats, Options{
 		Name:        name,
-		ReadBuf:     d.readBuf,
+		Force:       d.Options.Force,
 		FillGaps:    false,
 		IsRoot:      false,
 		Range:       ranges.Range{Start: d.Pos(), Len: d.BitsLeft()},
 		FormatInArg: inArg,
+		ReadBuf:     d.readBuf,
 	})
 	if dv == nil || dv.Errors() != nil {
 		return nil, nil, err
@@ -699,11 +709,12 @@ func (d *D) FieldFormat(name string, formats []*Format, inArg interface{}) (*Val
 func (d *D) FieldTryFormatLen(name string, nBits int64, formats []*Format, inArg interface{}) (*Value, interface{}, error) {
 	dv, v, err := decode(d.Ctx, d.bitBuf, formats, Options{
 		Name:        name,
-		ReadBuf:     d.readBuf,
+		Force:       d.Options.Force,
 		FillGaps:    true,
 		IsRoot:      false,
 		Range:       ranges.Range{Start: d.Pos(), Len: nBits},
 		FormatInArg: inArg,
+		ReadBuf:     d.readBuf,
 	})
 	if dv == nil || dv.Errors() != nil {
 		return nil, nil, err
@@ -729,11 +740,12 @@ func (d *D) FieldFormatLen(name string, nBits int64, formats []*Format, inArg in
 func (d *D) FieldTryFormatRange(name string, firstBit int64, nBits int64, formats []*Format, inArg interface{}) (*Value, interface{}, error) {
 	dv, v, err := decode(d.Ctx, d.bitBuf, formats, Options{
 		Name:        name,
-		ReadBuf:     d.readBuf,
+		Force:       d.Options.Force,
 		FillGaps:    true,
 		IsRoot:      false,
 		Range:       ranges.Range{Start: firstBit, Len: nBits},
 		FormatInArg: inArg,
+		ReadBuf:     d.readBuf,
 	})
 	if dv == nil || dv.Errors() != nil {
 		return nil, nil, err
@@ -756,10 +768,11 @@ func (d *D) FieldFormatRange(name string, firstBit int64, nBits int64, formats [
 func (d *D) FieldTryFormatBitBuf(name string, bb *bitio.Buffer, formats []*Format, inArg interface{}) (*Value, interface{}, error) {
 	dv, v, err := decode(d.Ctx, bb, formats, Options{
 		Name:        name,
-		ReadBuf:     d.readBuf,
+		Force:       d.Options.Force,
 		FillGaps:    true,
 		IsRoot:      true,
 		FormatInArg: inArg,
+		ReadBuf:     d.readBuf,
 	})
 	if dv == nil || dv.Errors() != nil {
 		return nil, nil, err

pkg/decode/scalar.go

@@ -226,10 +226,9 @@ func (d *D) BitBufValidateIsZero(s Scalar) (Scalar, error) {
 }
 
 // TODO: generate?
-func (d *D) assertRaw(assert bool, bss ...[]byte) func(s Scalar) (Scalar, error) {
+func (d *D) assertBitBuf(assert bool, bss ...[]byte) func(s Scalar) (Scalar, error) {
 	return func(s Scalar) (Scalar, error) {
-		// TODO: check type assert?
-		ab, err := s.Actual.(*bitio.Buffer).Bytes()
+		ab, err := s.ActualBitBuf().Bytes()
 		if err != nil {
 			return s, err
 		}
@@ -240,18 +239,18 @@ func (d *D) assertRaw(assert bool, bss ...[]byte) func(s Scalar) (Scalar, error)
 			}
 		}
 		s.Description = "invalid"
-		if assert {
+		if assert && !d.Options.Force {
 			return s, errors.New("failed to validate raw")
 		}
 		return s, nil
 	}
 }
 
-func (d *D) AssertRaw(bss ...[]byte) func(s Scalar) (Scalar, error) {
-	return d.assertRaw(true, bss...)
+func (d *D) AssertBitBuf(bss ...[]byte) func(s Scalar) (Scalar, error) {
+	return d.assertBitBuf(true, bss...)
 }
-func (d *D) ValidateRaw(bss ...[]byte) func(s Scalar) (Scalar, error) {
-	return d.assertRaw(false, bss...)
+func (d *D) ValidateBitBuf(bss ...[]byte) func(s Scalar) (Scalar, error) {
+	return d.assertBitBuf(false, bss...)
 }
 
 func (d *D) TryFieldValue(name string, fn func() (*Value, error)) (*Value, error) {