fq: Add truncate array support to dump/display

wader · Sep 12, 2021 · b849895 · b849895
1 parent ba273be
commit b849895
Show file tree

Hide file tree

Showing 10 changed files with 408 additions and 60 deletions.
diff --git a/doc/usage.md b/doc/usage.md
@@ -18,6 +18,7 @@ Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
 --null-output,-0        Null byte between outputs
 --option,-o KEY=VALUE   Set option, eg: color=true
                           addrbase=16
+                          arraytruncate=50
                           bitsformat=snippet
                           bytecolors=0-0xff=brightwhite,0=brightblack,32-126:9-13=white
                           color=false
@@ -100,8 +101,9 @@ notable is support for arbitrary-precision integers.
 - `open` open file for reading
 - `probe` or `decode` try to automatically detect format and decode
 - `mp3`, `matroska`, ..., `<name>`, `decode([name])` try decode as format
-- `d`/`display` display value
-- `v`/`verbose` display value verbosely
+- `d`/`display` display value and truncate long arrays
+- `f`/`full` display value and don't truncate arrays
+- `v`/`verbose` display value verbosely and don't truncate array
 - `p`/`preview` show preview of field tree
 - `hd`/`hexdump` hexdump value
 - `repl` nested REPL

diff --git a/pkg/decode/value.go b/pkg/decode/value.go
@@ -56,6 +56,7 @@ type Value struct {
 type WalkFn func(v *Value, rootV *Value, depth int, rootDepth int) error
 
 var ErrWalkSkipChildren = errors.New("skip children")
+var ErrWalkBreak = errors.New("break")
 var ErrWalkStop = errors.New("stop")
 
 func (v *Value) walk(preOrder bool, fn WalkFn) error {
@@ -85,12 +86,18 @@ func (v *Value) walk(preOrder bool, fn WalkFn) error {
 		case Struct:
 			for _, wv := range v {
 				if err := walkFn(wv, rootV, depth+1, rootDepth+rootDepthDelta); err != nil {
+					if errors.Is(err, ErrWalkBreak) {
+						break
+					}
 					return err
 				}
 			}
 		case Array:
 			for _, wv := range v {
 				if err := walkFn(wv, rootV, depth+1, rootDepth+rootDepthDelta); err != nil {
+					if errors.Is(err, ErrWalkBreak) {
+						break
+					}
 					return err
 				}
 			}

diff --git a/pkg/interp/dump.go b/pkg/interp/dump.go
@@ -52,8 +52,12 @@ func dumpEx(v *decode.Value, cw *columnwriter.Writer, depth int, rootV *decode.V
 	}
 
 	isInArray := false
+	inArrayLen := 0
 	if v.Parent != nil {
-		_, isInArray = v.Parent.V.(decode.Array)
+		if da, ok := v.Parent.V.(decode.Array); ok {
+			isInArray = true
+			inArrayLen = len(da)
+		}
 	}
 
 	nameV := v
@@ -86,6 +90,19 @@ func dumpEx(v *decode.Value, cw *columnwriter.Writer, depth int, rootV *decode.V
 		}
 	}
 
+	if opts.ArrayTruncate != 0 && depth != 0 && isInArray && v.Index >= opts.ArrayTruncate {
+		columns()
+		cfmt(colField, "%s%s%s:%s%s: ...",
+			indent,
+			deco.Index.F("["),
+			deco.Number.F(strconv.Itoa(v.Index)),
+			deco.Number.F(strconv.Itoa(inArrayLen-1)),
+			deco.Index.F("]"),
+		)
+		cw.Flush()
+		return decode.ErrWalkBreak
+	}
+
 	cfmt(colField, "%s%s", indent, name)
 	if isInArray {
 		cfmt(colField, "%s%s%s", deco.Index.F("["), deco.Number.F(strconv.Itoa(v.Index)), deco.Index.F("]"))
@@ -95,6 +112,8 @@ func dumpEx(v *decode.Value, cw *columnwriter.Writer, depth int, rootV *decode.V
 		cfmt(colField, " %s", v.Name)
 	}
 
+	// TODO: cleanup map[string]interface{} []interface{} or json format
+	// dump should use some internal interface instead?
 	switch v.V.(type) {
 	case decode.Struct, map[string]interface{}:
 		cfmt(colField, " %s", deco.Object.F("{}"))

diff --git a/pkg/interp/funcs.go b/pkg/interp/funcs.go
@@ -54,7 +54,8 @@ func (i *Interp) makeFunctions(registry *registry.Registry) []Function {
 
 		{[]string{"format"}, 0, 0, i.format, nil},
 		{[]string{"display", "d"}, 0, 1, nil, i.makeDisplayFn(nil)},
-		{[]string{"verbose", "v"}, 0, 1, nil, i.makeDisplayFn(map[string]interface{}{"verbose": true})},
+		{[]string{"full", "f"}, 0, 1, nil, i.makeDisplayFn(map[string]interface{}{"arraytruncate": 0})},
+		{[]string{"verbose", "v"}, 0, 1, nil, i.makeDisplayFn(map[string]interface{}{"arraytruncate": 0, "verbose": true})},
 		{[]string{"preview", "p"}, 0, 1, nil, i.preview},
 		{[]string{"hexdump", "hd", "h"}, 0, 1, nil, i.hexdump},
 

diff --git a/pkg/interp/interp.go b/pkg/interp/interp.go
@@ -787,6 +787,7 @@ func (i *Interp) EvalFuncValues(ctx context.Context, mode RunMode, c interface{}
 
 type Options struct {
 	Depth          int
+	ArrayTruncate  int
 	Verbose        bool
 	DecodeProgress bool
 	Color          bool
@@ -813,6 +814,9 @@ func mapSetOptions(d *Options, m map[string]interface{}) {
 	if v, ok := m["depth"]; ok {
 		d.Depth = num.MaxInt(0, toIntZ(v))
 	}
+	if v, ok := m["arraytruncate"]; ok {
+		d.ArrayTruncate = num.MaxInt(0, toIntZ(v))
+	}
 	if v, ok := m["verbose"]; ok {
 		d.Verbose = toBoolZ(v)
 	}

diff --git a/pkg/interp/interp.jq b/pkg/interp/interp.jq
@@ -48,6 +48,7 @@ def _obj_to_csv_kv:
 def _build_default_options:
   {
     addrbase:       16,
+    arraytruncate:  50,
     bitsformat:     "snippet",
     bytecolors:     "0-0xff=brightwhite,0=brightblack,32-126:9-13=white",
     color:          (tty.is_terminal and env.CLICOLOR != null),
@@ -105,6 +106,7 @@ def _tostring:
 def _to_options:
   ( {
       addrbase:        (.addrbase | _tonumber),
+      arraytruncate:   (.arraytruncate | _tonumber),
       bitsformat:      (.bitsformat | _tostring),
       bytecolors:      (.bytecolors | _tostring),
       color:           (.color | _toboolean),

diff --git a/pkg/interp/testdata/args.fqtest b/pkg/interp/testdata/args.fqtest
@@ -20,6 +20,7 @@ Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
 --null-output,-0        Null byte between outputs
 --option,-o KEY=VALUE   Set option, eg: color=true
                           addrbase=16
+                          arraytruncate=50
                           bitsformat=snippet
                           bytecolors=0-0xff=brightwhite,0=brightblack,32-126:9-13=white
                           color=false