fix non-ascii characters handling in to_hex and to_base64 functions

to_hex and to_base64 functions were expecting `string` as an input type, which caused the value to be CastFn'ed to string, which in turn resulted in raw bytes being cast to []rune (in makeDecodeValueOut). This operation replaces invalid UTF-8 bytes with 0xFFFD, which then were passed on to the hex/base64 encoders, resulting in incorrect output. This patch fixes it by expecting `any` as an input type, which allows the function to correctly read raw bytes of the input data.
wader · May 16, 2023 · b0e4da2 · b0e4da2
1 parent 7eaddb8
commit b0e4da2
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 2 deletions.
diff --git a/format/text/encoding.go b/format/text/encoding.go
@@ -31,7 +31,7 @@ func init() {
 		}
 		return bb
 	})
-	interp.RegisterFunc0("to_hex", func(_ *interp.Interp, c string) any {
+	interp.RegisterFunc0("to_hex", func(_ *interp.Interp, c any) any {
 		br, err := interp.ToBitReader(c)
 		if err != nil {
 			return err
@@ -73,7 +73,7 @@ func init() {
 	type toBase64Opts struct {
 		Encoding string
 	}
-	interp.RegisterFunc1("_to_base64", func(_ *interp.Interp, c string, opts toBase64Opts) any {
+	interp.RegisterFunc1("_to_base64", func(_ *interp.Interp, c any, opts toBase64Opts) any {
 		br, err := interp.ToBitReader(c)
 		if err != nil {
 			return err

diff --git a/format/text/testdata/random.bin.gz b/format/text/testdata/random.bin.gz
diff --git a/format/text/testdata/to_base64.fqtest b/format/text/testdata/to_base64.fqtest
@@ -0,0 +1,2 @@
+$ fq '.uncompressed | to_base64' random.bin.gz -r
+8Vr6cUH6erA=
diff --git a/format/text/testdata/to_hex.fqtest b/format/text/testdata/to_hex.fqtest
@@ -0,0 +1,2 @@
+$ fq '.uncompressed | tohex' random.bin.gz -r
+f15afa7141fa7ab0