interp: Add skip_gaps option for tovalue/-V

Skips gap fields in struct and arrays. Gaps fields are bit ranges that a decoder did not add any fields for. Note that skipping gaps in arrays will affect indexes.
wader · Apr 30, 2023 · c512713 · c512713
1 parent 1bdf1df
commit c512713
Show file tree

Hide file tree

Showing 9 changed files with 96 additions and 31 deletions.
diff --git a/doc/usage.md b/doc/usage.md
@@ -796,16 +796,36 @@ fq has some general options in addition to decode and decoders specific options.
 
 `<value>` is fuzzily parsed based on the type of the option. Ex: a string can be specified as `-o name=string` or `-o name="string"`.
 
-### `bits_format`
+### `-o bits_format=<string>`
 
-How to represent raw bits as JSON.
+How to represent raw binary as JSON.
 
-- `-o bits_foramt=string` String with raw bytes (zero bit padded). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
+- `-o bits_foramt=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
 - `-o bits_format=md5` MD5 hex string (zero bit padded).
 - `-o bits_format=base64` Base64 string.
 - `-p bits_foramt=truncate` Truncated string.
 - `-o bits_format=snippet` Truncated Base64 string prefixed with bit length.
 
+```sh
+$ fq -V -o bits_format=base64 . file`
+```
+In query
+```jq
+tovalue({bits_format: "md5"})
+```
+
+### `-o skip_gaps=<boolean>`
+
+Skip gaps fields (`gap0` etc) when using `tovalue` or `-V`. Note that this might affect array indexes if one more more gaps fields are skipped in an array.
+
+```sh
+$ fq -V -o skip_gaps=true . file`
+```
+In query
+```jq
+tovalue({skip_gaps: true})
+```
+
 ## Color and unicode output
 
 fq by default tries to use colors if possible, this can be disabled with `-M`. You can also

diff --git a/pkg/interp/decode.go b/pkg/interp/decode.go
@@ -163,9 +163,12 @@ func (i *Interp) _registry(c any) any {
 	}
 }
 
-func (i *Interp) _toValue(c any, opts map[string]any) any {
+func (i *Interp) _toValue(c any, om map[string]any) any {
 	return toValue(
-		func() Options { return OptionsFromValue(opts) },
+		func() *Options {
+			opts := OptionsFromValue(om)
+			return &opts
+		},
 		c,
 	)
 }
@@ -307,7 +310,7 @@ func valueHas(key any, a func(name string) any, b func(key any) any) any {
 
 // TODO: make more efficient somehow? shallow values but might be hard
 // when things like tovalue.key should behave like a jq value and not a decode value etc
-func toValue(optsFn func() Options, v any) any {
+func toValue(optsFn func() *Options, v any) any {
 	nv, _ := gojqex.ToGoJQValueFn(v, func(v any) (any, bool) {
 		switch v := v.(type) {
 		case JQValueEx:
@@ -540,7 +543,7 @@ func (dvb decodeValueBase) JQValueKey(name string) any {
 	case "_gap":
 		switch vv := dv.V.(type) {
 		case Scalarable:
-			return vv.ScalarGap()
+			return vv.ScalarIsGap()
 		default:
 			return false
 		}
@@ -606,7 +609,7 @@ func (v decodeValue) JQValueKey(name string) any {
 func (v decodeValue) JQValueHas(key any) any {
 	return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
 }
-func (v decodeValue) JQValueToGoJQEx(optsFn func() Options) any {
+func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
 	if !v.bitsFormat {
 		return v.JQValueToGoJQ()
 	}
@@ -695,13 +698,26 @@ func (v ArrayDecodeValue) JQValueHas(key any) any {
 			return intKey >= 0 && intKey < len(v.Compound.Children)
 		})
 }
-func (v ArrayDecodeValue) JQValueToGoJQ() any {
-	vs := make([]any, len(v.Compound.Children))
-	for i, f := range v.Compound.Children {
-		vs[i] = makeDecodeValue(f, decodeValueValue)
+func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
+	opts := optsFn()
+
+	vs := make([]any, 0, len(v.Compound.Children))
+	for _, f := range v.Compound.Children {
+		switch s := f.V.(type) {
+		case Scalarable:
+			if s.ScalarIsGap() && opts.SkipGaps {
+				// skip, note for arrays this will affect indexes
+				continue
+			}
+		}
+
+		vs = append(vs, makeDecodeValue(f, decodeValueValue))
 	}
 	return vs
 }
+func (v ArrayDecodeValue) JQValueToGoJQ() any {
+	return v.JQValueToGoJQEx(func() *Options { return &Options{} })
+}
 
 // decode value struct
 
@@ -767,10 +783,22 @@ func (v StructDecodeValue) JQValueHas(key any) any {
 		},
 	)
 }
-func (v StructDecodeValue) JQValueToGoJQ() any {
+func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
+	opts := optsFn()
+
 	vm := make(map[string]any, len(v.Compound.Children))
 	for _, f := range v.Compound.Children {
+		switch s := f.V.(type) {
+		case Scalarable:
+			if s.ScalarIsGap() && opts.SkipGaps {
+				continue
+			}
+		}
+
 		vm[f.Name] = makeDecodeValue(f, decodeValueValue)
 	}
 	return vm
 }
+func (v StructDecodeValue) JQValueToGoJQ() any {
+	return v.JQValueToGoJQEx(func() *Options { return &Options{} })
+}
diff --git a/pkg/interp/interp.go b/pkg/interp/interp.go
@@ -82,7 +82,7 @@ type Scalarable interface {
 	ScalarValue() any
 	ScalarSym() any
 	ScalarDescription() string
-	ScalarGap() bool
+	ScalarIsGap() bool
 	ScalarDisplayFormat() scalar.DisplayFormat
 }
 
@@ -210,7 +210,7 @@ type Display interface {
 
 type JQValueEx interface {
 	gojq.JQValue
-	JQValueToGoJQEx(optsFn func() Options) any
+	JQValueToGoJQEx(optsFn func() *Options) any
 }
 
 func valuePath(v *decode.Value) []any {
@@ -682,7 +682,7 @@ func (i *Interp) _printColorJSON(c any, v any) gojq.Iter {
 		Color:   opts.Color,
 		Tab:     false,
 		Indent:  indent,
-		ValueFn: func(v any) any { return toValue(func() Options { return opts }, v) },
+		ValueFn: func(v any) any { return toValue(func() *Options { return &opts }, v) },
 		Colors: colorjson.Colors{
 			Reset:     []byte(ansi.Reset.SetString),
 			Null:      []byte(opts.Decorator.Null.SetString),
@@ -1034,6 +1034,7 @@ type Options struct {
 	DisplayBytes int
 	Addrbase     int
 	Sizebase     int
+	SkipGaps     bool
 
 	Decorator    Decorator
 	BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)

diff --git a/pkg/interp/options.jq b/pkg/interp/options.jq
@@ -59,6 +59,7 @@ def _opt_build_default_fixed:
       raw_output:         ($stdout.is_terminal | not),
       raw_string:         false,
       repl:               false,
+      skip_gaps:          false,
       sizebase:           10,
       show_formats:       false,
       show_help:          false,
@@ -101,9 +102,10 @@ def _opt_options:
     raw_output:         "boolean",
     raw_string:         "boolean",
     repl:               "boolean",
-    sizebase:           "number",
     show_formats:       "boolean",
     show_help:          "boolean",
+    sizebase:           "number",
+    skip_gaps:          "boolean",
     slurp:              "boolean",
     string_input:       "boolean",
     unicode:            "boolean",

diff --git a/pkg/interp/testdata/args.fqtest b/pkg/interp/testdata/args.fqtest
@@ -99,6 +99,7 @@ repl                false
 show_formats        false
 show_help           options
 sizebase            10
+skip_gaps           false
 slurp               false
 string_input        false
 unicode             false

diff --git a/pkg/interp/testdata/options.fqtest b/pkg/interp/testdata/options.fqtest
@@ -83,6 +83,7 @@ $ fq -n options
   "show_formats": false,
   "show_help": false,
   "sizebase": 10,
+  "skip_gaps": false,
   "slurp": false,
   "string_input": false,
   "unicode": false,

diff --git a/pkg/interp/testdata/tovalue.fqtest b/pkg/interp/testdata/tovalue.fqtest
@@ -2,9 +2,21 @@
 $ fq -i . test.mp3
 mp3> .headers[0] | tovalue.header.magic
 "ID3"
+mp3> [tobytes, "abc"] | mp3 | tovalue, tovalue({skip_gaps: true}) | keys
+[
+  "footers",
+  "frames",
+  "gap0",
+  "headers"
+]
+[
+  "footers",
+  "frames",
+  "headers"
+]
+mp3> "abc" | mpeg_ts | tovalue, tovalue({skip_gaps: true}) | keys
+[
+  "gap0"
+]
+[]
 mp3> ^D
-$ fq -i
-null> "aaa" | mp3_frame | .gap0 | tovalue, tovalue({sizebase: 2})
-"aaa"
-"aaa"
-null> ^D
diff --git a/pkg/scalar/scalar_gen.go b/pkg/scalar/scalar_gen.go
@@ -27,7 +27,7 @@ func (s Any) ScalarValue() any {
 }
 func (s Any) ScalarSym() any                     { return s.Sym }
 func (s Any) ScalarDescription() string          { return s.Description }
-func (s Any) ScalarGap() bool                    { return s.Gap }
+func (s Any) ScalarIsGap() bool                  { return s.Gap }
 func (s Any) ScalarDisplayFormat() DisplayFormat { return 0 }
 
 func AnyActual(v any) AnyMapper {
@@ -224,7 +224,7 @@ func (s BigInt) ScalarValue() any {
 }
 func (s BigInt) ScalarSym() any                     { return s.Sym }
 func (s BigInt) ScalarDescription() string          { return s.Description }
-func (s BigInt) ScalarGap() bool                    { return s.Gap }
+func (s BigInt) ScalarIsGap() bool                  { return s.Gap }
 func (s BigInt) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
 
 func BigIntActual(v *big.Int) BigIntMapper {
@@ -420,7 +420,7 @@ func (s BitBuf) ScalarValue() any {
 }
 func (s BitBuf) ScalarSym() any                     { return s.Sym }
 func (s BitBuf) ScalarDescription() string          { return s.Description }
-func (s BitBuf) ScalarGap() bool                    { return s.Gap }
+func (s BitBuf) ScalarIsGap() bool                  { return s.Gap }
 func (s BitBuf) ScalarDisplayFormat() DisplayFormat { return 0 }
 
 func BitBufActual(v bitio.ReaderAtSeeker) BitBufMapper {
@@ -616,7 +616,7 @@ func (s Bool) ScalarValue() any {
 }
 func (s Bool) ScalarSym() any                     { return s.Sym }
 func (s Bool) ScalarDescription() string          { return s.Description }
-func (s Bool) ScalarGap() bool                    { return s.Gap }
+func (s Bool) ScalarIsGap() bool                  { return s.Gap }
 func (s Bool) ScalarDisplayFormat() DisplayFormat { return 0 }
 
 func BoolActual(v bool) BoolMapper {
@@ -812,7 +812,7 @@ func (s Flt) ScalarValue() any {
 }
 func (s Flt) ScalarSym() any                     { return s.Sym }
 func (s Flt) ScalarDescription() string          { return s.Description }
-func (s Flt) ScalarGap() bool                    { return s.Gap }
+func (s Flt) ScalarIsGap() bool                  { return s.Gap }
 func (s Flt) ScalarDisplayFormat() DisplayFormat { return 0 }
 
 func FltActual(v float64) FltMapper {
@@ -1009,7 +1009,7 @@ func (s Sint) ScalarValue() any {
 }
 func (s Sint) ScalarSym() any                     { return s.Sym }
 func (s Sint) ScalarDescription() string          { return s.Description }
-func (s Sint) ScalarGap() bool                    { return s.Gap }
+func (s Sint) ScalarIsGap() bool                  { return s.Gap }
 func (s Sint) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
 
 func SintActual(v int64) SintMapper {
@@ -1205,7 +1205,7 @@ func (s Str) ScalarValue() any {
 }
 func (s Str) ScalarSym() any                     { return s.Sym }
 func (s Str) ScalarDescription() string          { return s.Description }
-func (s Str) ScalarGap() bool                    { return s.Gap }
+func (s Str) ScalarIsGap() bool                  { return s.Gap }
 func (s Str) ScalarDisplayFormat() DisplayFormat { return 0 }
 
 func StrActual(v string) StrMapper {
@@ -1402,7 +1402,7 @@ func (s Uint) ScalarValue() any {
 }
 func (s Uint) ScalarSym() any                     { return s.Sym }
 func (s Uint) ScalarDescription() string          { return s.Description }
-func (s Uint) ScalarGap() bool                    { return s.Gap }
+func (s Uint) ScalarIsGap() bool                  { return s.Gap }
 func (s Uint) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
 
 func UintActual(v uint64) UintMapper {

diff --git a/pkg/scalar/scalar_gen.go.tmpl b/pkg/scalar/scalar_gen.go.tmpl
@@ -31,7 +31,7 @@ import (
 	}
 	func (s {{$name}}) ScalarSym() any { return s.Sym }
 	func (s {{$name}}) ScalarDescription() string { return s.Description }
-	func (s {{$name}}) ScalarGap() bool { return s.Gap }
+	func (s {{$name}}) ScalarIsGap() bool { return s.Gap }
 	{{- if $t.display_format}}
 	func (s {{$name}}) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
 	{{- else }}