From 222cd88b770561bca328457a90c5c00b427ae35c Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Thu, 11 May 2023 17:48:02 +0200 Subject: [PATCH] bits,bytes: Behave as binary instead of raw decode value I think this is closer to what one would expect. With this fq -d bytes 'scan("...")' will match as a binary and also return binaries, before as raw decode value one would have to use tobytes to get the binary. --- format/bits/testdata/test.fqtest | 16 ++++++++++++---- pkg/interp/binary.go | 3 +++ pkg/interp/binary.jq | 28 ++++++++++++++++++---------- pkg/interp/decode.go | 5 +---- pkg/interp/repl.jq | 8 ++++++-- pkg/interp/testdata/repl.fqtest | 2 +- 6 files changed, 41 insertions(+), 21 deletions(-) diff --git a/format/bits/testdata/test.fqtest b/format/bits/testdata/test.fqtest index a849289f9..e096a3eeb 100644 --- a/format/bits/testdata/test.fqtest +++ b/format/bits/testdata/test.fqtest @@ -1,14 +1,22 @@ /hello: hello -$ fq -d bits '., .size, .[8:-8]' hello +$ fq -d bits '., .size, .[8:-8], scan("he|lo")' hello |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| -0x0|68 65 6c 6c 6f 0a| |hello.| |.: raw bits (bits) +0x0|68 65 6c 6c 6f 0a| |hello.| |.: raw bits 0x0-0x5.7 (6) 48 |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| 0x0| 65 6c 6c 6f | ello |.: raw bits 0x1-0x4.7 (4) -$ fq -d bytes '., .size, .[1:-1]' hello |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| -0x0|68 65 6c 6c 6f 0a| |hello.| |.: raw bits (bytes) +0x0|68 65 |he |.: raw bits 0x0-0x1.7 (2) + |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| +0x0| 6c 6f | lo |.: raw bits 0x3-0x4.7 (2) +$ fq -d bytes '., .size, .[1:-1], scan("he|lo")' hello + |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| +0x0|68 65 6c 6c 6f 0a| |hello.| |.: raw bits 0x0-0x5.7 (6) 6 |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| 0x0| 65 6c 6c 6f | ello |.: raw bits 0x1-0x4.7 (4) + |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| +0x0|68 65 |he |.: raw bits 0x0-0x1.7 (2) + |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| +0x0| 6c 6f | lo |.: raw bits 0x3-0x4.7 (2) diff --git a/pkg/interp/binary.go b/pkg/interp/binary.go index 9eacd3ff6..82f29e776 100644 --- a/pkg/interp/binary.go +++ b/pkg/interp/binary.go @@ -299,6 +299,7 @@ func (Binary) ExtKeys() []string { "size", "start", "stop", + "unit", } } @@ -368,6 +369,8 @@ func (b Binary) JQValueKey(name string) any { stopUnits++ } return new(big.Int).SetInt64(stopUnits) + case "unit": + return b.unit } return nil } diff --git a/pkg/interp/binary.jq b/pkg/interp/binary.jq index 57a4b9e25..2e5c8dce6 100644 --- a/pkg/interp/binary.jq +++ b/pkg/interp/binary.jq @@ -18,6 +18,14 @@ def _binary_or_orig(bfn; fn): if _exttype == "binary" then bfn else fn end; +def _bytes_or_orig(bfn; fn): + _binary_or_orig( + # convert to bytes if bits + ( if .unit != 8 then tobytesrange end + | bfn + ); + fn + ); def _orig_explode: explode; def explode: _binary_or_orig([.[range(.size)]]; _orig_explode); @@ -38,8 +46,8 @@ def _splits_binary($regex; $flags): end ) ); -def splits($val): _binary_or_orig(_splits_binary($val; "g"); _orig_splits($val)); -def splits($regex; $flags): _binary_or_orig(_splits_binary($regex; "g"+$flags); _orig_splits($regex; $flags)); +def splits($val): _bytes_or_orig(_splits_binary($val; "g"); _orig_splits($val)); +def splits($regex; $flags): _bytes_or_orig(_splits_binary($regex; "g"+$flags); _orig_splits($regex; $flags)); def _orig_split($val): split($val); def _orig_split($regex; $flags): split($regex; $flags); @@ -53,13 +61,13 @@ def _test_binary($regex; $flags): ( isempty(_match_binary($regex; $flags)) | not ); -def test($val): _binary_or_orig(_test_binary($val; ""); _orig_test($val)); -def test($regex; $flags): _binary_or_orig(_test_binary($regex; $flags); _orig_test($regex; $flags)); +def test($val): _bytes_or_orig(_test_binary($val; ""); _orig_test($val)); +def test($regex; $flags): _bytes_or_orig(_test_binary($regex; $flags); _orig_test($regex; $flags)); def _orig_match($val): match($val); def _orig_match($regex; $flags): match($regex; $flags); -def match($val): _binary_or_orig(_match_binary($val; ""); _orig_match($val)); -def match($regex; $flags): _binary_or_orig(_match_binary($regex; $flags); _orig_match($regex; $flags)); +def match($val): _bytes_or_orig(_match_binary($val; ""); _orig_match($val)); +def match($regex; $flags): _bytes_or_orig(_match_binary($regex; $flags); _orig_match($regex; $flags)); def _orig_capture($val): capture($val); def _orig_capture($regex; $flags): capture($regex; $flags); @@ -74,8 +82,8 @@ def _capture_binary($regex; $flags): ) | from_entries ); -def capture($val): _binary_or_orig(_capture_binary($val; ""); _orig_capture($val)); -def capture($regex; $flags): _binary_or_orig(_capture_binary($regex; $flags); _orig_capture($regex; $flags)); +def capture($val): _bytes_or_orig(_capture_binary($val; ""); _orig_capture($val)); +def capture($regex; $flags): _bytes_or_orig(_capture_binary($regex; $flags); _orig_capture($regex; $flags)); def _orig_scan($val): scan($val); def _orig_scan($regex; $flags): scan($regex; $flags); @@ -84,5 +92,5 @@ def _scan_binary($regex; $flags): | _match_binary($regex; $flags) | $b[.offset:.offset+.length] ); -def scan($val): _binary_or_orig(_scan_binary($val; "g"); _orig_scan($val)); -def scan($regex; $flags): _binary_or_orig(_scan_binary($regex; "g"+$flags); _orig_scan($regex; $flags)); +def scan($val): _bytes_or_orig(_scan_binary($val; "g"); _orig_scan($val)); +def scan($regex; $flags): _bytes_or_orig(_scan_binary($regex; "g"+$flags); _orig_scan($regex; $flags)); diff --git a/pkg/interp/decode.go b/pkg/interp/decode.go index dd3bf740f..ebc56291f 100644 --- a/pkg/interp/decode.go +++ b/pkg/interp/decode.go @@ -434,10 +434,7 @@ func makeDecodeValueOut(dv *decode.Value, kind decodeValueKind, out any) any { decodeValueBase: decodeValueBase{dv: dv}, } case Binary: - return decodeValue{ - JQValue: vvv, - decodeValueBase: decodeValueBase{dv: dv}, - } + return vvv default: panic(fmt.Sprintf("unreachable vv %#+v", vvv)) diff --git a/pkg/interp/repl.jq b/pkg/interp/repl.jq index 3faa2bc04..829eab167 100644 --- a/pkg/interp/repl.jq +++ b/pkg/interp/repl.jq @@ -136,7 +136,7 @@ def _prompt($opts): , "]" , if length > 1 then ( ("[" | _ansi_if($opts; "array")) - , ("0" | _ansi_if($opts; "number")) + , ("0" | _ansi_if($opts; "number")) , ":" , (length | tostring | _ansi_if($opts; "number")) , ("]" | _ansi_if($opts; "array")) @@ -152,7 +152,11 @@ def _prompt($opts): + if $c._error then "!" else "" end ) else - ($c | type) + ( $c + | if _is_decode_value then type + else (_exttype // type) + end + ) end ) | _ansi_if($opts; "prompt_value") end; diff --git a/pkg/interp/testdata/repl.fqtest b/pkg/interp/testdata/repl.fqtest index 44728d4c5..43b5ef24a 100644 --- a/pkg/interp/testdata/repl.fqtest +++ b/pkg/interp/testdata/repl.fqtest @@ -108,7 +108,7 @@ json!> ^D $ fq -i -n '"[]" | json' json> ^D $ fq -i -d bytes . test.mp3 -bytes> ^D +binary> ^D $ fq -n repl exitcode: 3 stderr: