bits,bytes: Behave as binary instead of raw decode value

I think this is closer to what one would expect. With this fq -d bytes 'scan("...")' will match as a binary and also return binaries, before as raw decode value one would have to use tobytes to get the binary.
wader · May 11, 2023 · 222cd88 · 222cd88
1 parent dc91e5e
commit 222cd88
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 21 deletions.
diff --git a/format/bits/testdata/test.fqtest b/format/bits/testdata/test.fqtest
@@ -1,14 +1,22 @@
 /hello:
 hello
-$ fq -d bits '., .size, .[8:-8]' hello
+$ fq -d bits '., .size, .[8:-8], scan("he|lo")' hello
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|68 65 6c 6c 6f 0a|                             |hello.|         |.: raw bits (bits)
+0x0|68 65 6c 6c 6f 0a|                             |hello.|         |.: raw bits 0x0-0x5.7 (6)
 48
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|   65 6c 6c 6f                                 | ello           |.: raw bits 0x1-0x4.7 (4)
-$ fq -d bytes '., .size, .[1:-1]' hello
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|68 65 6c 6c 6f 0a|                             |hello.|         |.: raw bits (bytes)
+0x0|68 65                                          |he              |.: raw bits 0x0-0x1.7 (2)
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|         6c 6f                                 |   lo           |.: raw bits 0x3-0x4.7 (2)
+$ fq -d bytes '., .size, .[1:-1], scan("he|lo")' hello
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|68 65 6c 6c 6f 0a|                             |hello.|         |.: raw bits 0x0-0x5.7 (6)
 6
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|   65 6c 6c 6f                                 | ello           |.: raw bits 0x1-0x4.7 (4)
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|68 65                                          |he              |.: raw bits 0x0-0x1.7 (2)
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|         6c 6f                                 |   lo           |.: raw bits 0x3-0x4.7 (2)
diff --git a/pkg/interp/binary.go b/pkg/interp/binary.go
@@ -299,6 +299,7 @@ func (Binary) ExtKeys() []string {
 		"size",
 		"start",
 		"stop",
+		"unit",
 	}
 }
 
@@ -368,6 +369,8 @@ func (b Binary) JQValueKey(name string) any {
 			stopUnits++
 		}
 		return new(big.Int).SetInt64(stopUnits)
+	case "unit":
+		return b.unit
 	}
 	return nil
 }

diff --git a/pkg/interp/binary.jq b/pkg/interp/binary.jq
@@ -18,6 +18,14 @@ def _binary_or_orig(bfn; fn):
   if _exttype == "binary" then bfn
   else fn
   end;
+def _bytes_or_orig(bfn; fn):
+  _binary_or_orig(
+    # convert to bytes if bits
+    ( if .unit != 8 then tobytesrange end
+    | bfn
+    );
+    fn
+  );
 
 def _orig_explode: explode;
 def explode: _binary_or_orig([.[range(.size)]]; _orig_explode);
@@ -38,8 +46,8 @@ def _splits_binary($regex; $flags):
       end
     )
   );
-def splits($val): _binary_or_orig(_splits_binary($val; "g"); _orig_splits($val));
-def splits($regex; $flags): _binary_or_orig(_splits_binary($regex; "g"+$flags); _orig_splits($regex; $flags));
+def splits($val): _bytes_or_orig(_splits_binary($val; "g"); _orig_splits($val));
+def splits($regex; $flags): _bytes_or_orig(_splits_binary($regex; "g"+$flags); _orig_splits($regex; $flags));
 
 def _orig_split($val): split($val);
 def _orig_split($regex; $flags): split($regex; $flags);
@@ -53,13 +61,13 @@ def _test_binary($regex; $flags):
   ( isempty(_match_binary($regex; $flags))
   | not
   );
-def test($val): _binary_or_orig(_test_binary($val; ""); _orig_test($val));
-def test($regex; $flags): _binary_or_orig(_test_binary($regex; $flags); _orig_test($regex; $flags));
+def test($val): _bytes_or_orig(_test_binary($val; ""); _orig_test($val));
+def test($regex; $flags): _bytes_or_orig(_test_binary($regex; $flags); _orig_test($regex; $flags));
 
 def _orig_match($val): match($val);
 def _orig_match($regex; $flags): match($regex; $flags);
-def match($val): _binary_or_orig(_match_binary($val; ""); _orig_match($val));
-def match($regex; $flags): _binary_or_orig(_match_binary($regex; $flags); _orig_match($regex; $flags));
+def match($val): _bytes_or_orig(_match_binary($val; ""); _orig_match($val));
+def match($regex; $flags): _bytes_or_orig(_match_binary($regex; $flags); _orig_match($regex; $flags));
 
 def _orig_capture($val): capture($val);
 def _orig_capture($regex; $flags): capture($regex; $flags);
@@ -74,8 +82,8 @@ def _capture_binary($regex; $flags):
     )
   | from_entries
   );
-def capture($val): _binary_or_orig(_capture_binary($val; ""); _orig_capture($val));
-def capture($regex; $flags): _binary_or_orig(_capture_binary($regex; $flags); _orig_capture($regex; $flags));
+def capture($val): _bytes_or_orig(_capture_binary($val; ""); _orig_capture($val));
+def capture($regex; $flags): _bytes_or_orig(_capture_binary($regex; $flags); _orig_capture($regex; $flags));
 
 def _orig_scan($val): scan($val);
 def _orig_scan($regex; $flags): scan($regex; $flags);
@@ -84,5 +92,5 @@ def _scan_binary($regex; $flags):
   | _match_binary($regex; $flags)
   | $b[.offset:.offset+.length]
   );
-def scan($val): _binary_or_orig(_scan_binary($val; "g"); _orig_scan($val));
-def scan($regex; $flags): _binary_or_orig(_scan_binary($regex; "g"+$flags); _orig_scan($regex; $flags));
+def scan($val): _bytes_or_orig(_scan_binary($val; "g"); _orig_scan($val));
+def scan($regex; $flags): _bytes_or_orig(_scan_binary($regex; "g"+$flags); _orig_scan($regex; $flags));
diff --git a/pkg/interp/decode.go b/pkg/interp/decode.go
@@ -434,10 +434,7 @@ func makeDecodeValueOut(dv *decode.Value, kind decodeValueKind, out any) any {
 				decodeValueBase: decodeValueBase{dv: dv},
 			}
 		case Binary:
-			return decodeValue{
-				JQValue:         vvv,
-				decodeValueBase: decodeValueBase{dv: dv},
-			}
+			return vvv
 
 		default:
 			panic(fmt.Sprintf("unreachable vv %#+v", vvv))

diff --git a/pkg/interp/repl.jq b/pkg/interp/repl.jq
@@ -136,7 +136,7 @@ def _prompt($opts):
       , "]"
       , if length > 1 then
           ( ("[" | _ansi_if($opts; "array"))
-          , ("0" |  _ansi_if($opts; "number"))
+          , ("0" | _ansi_if($opts; "number"))
           , ":"
           , (length | tostring | _ansi_if($opts; "number"))
           , ("]" | _ansi_if($opts; "array"))
@@ -152,7 +152,11 @@ def _prompt($opts):
           + if $c._error then "!" else "" end
           )
         else
-          ($c | type)
+          ( $c
+          | if _is_decode_value then type
+            else (_exttype // type)
+            end
+          )
         end
       ) | _ansi_if($opts; "prompt_value")
     end;

diff --git a/pkg/interp/testdata/repl.fqtest b/pkg/interp/testdata/repl.fqtest
@@ -108,7 +108,7 @@ json!> ^D
 $ fq -i -n '"[]" | json'
 json> ^D
 $ fq -i -d bytes . test.mp3
-bytes> ^D
+binary> ^D
 $ fq -n repl
 exitcode: 3
 stderr: