interp,format: Refactor registry usage and use function helpers

Move registry to interp and add support for functions and filesystems. This will be used later for allow formats to add own functions and fq code. Add gojqextra function helpers to have more comfortable API to add functions. Takes care of argument type casting and JQValue:s and some more things. Refactor interp package to use new function helper and registry. Probably fixes a bunch of JQValue bugs and other type errors. Refactor out some mpeg nal things to mpeg format. Refactor interp jq code into display.q and init.jq. Remove undocumented aes_ctr funciton, was a test. Hopefully will add more crypto things laster.
wader · Jul 16, 2022 · 1ddea1a · 1ddea1a
1 parent 88dc847
commit 1ddea1a
Show file tree

Hide file tree

Showing 135 changed files with 1,340 additions and 1,112 deletions.
diff --git a/dev/tmpl.go b/dev/tmpl.go
@@ -43,7 +43,7 @@ func main() {
 			min := toInt(args[0])
 			max := toInt(args[1])
 			var v []int
-			for i := min; i <= max; i++ {
+			for i := min; i < max; i++ {
 				v = append(v, i)
 			}
 
@@ -60,6 +60,16 @@ func main() {
 
 			return strings.Replace(s, o, n, -1), nil
 		},
+		"slice": func(args ...any) []any {
+			return args
+		},
+		"map": func(args ...any) map[any]any {
+			m := map[any]any{}
+			for i := 0; i < len(args)/2; i++ {
+				m[args[i*2]] = args[i*2+1]
+			}
+			return m
+		},
 	}
 
 	data := map[string]any{}

diff --git a/format/ape/apev2.go b/format/ape/apev2.go
@@ -4,14 +4,14 @@ package ape
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 )
 
 var imageFormat decode.Group
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.APEV2,
 		Description: "APEv2 metadata tag",
 		DecodeFn:    apev2Decode,

diff --git a/format/ar/ar.go b/format/ar/ar.go
@@ -2,15 +2,15 @@ package ar
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 var probeFormat decode.Group
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.AR,
 		Description: "Unix archive",
 		Groups:      []string{format.PROBE},

diff --git a/format/asn1/asn1_ber.go b/format/asn1/asn1_ber.go
@@ -22,17 +22,17 @@ import (
 	"strings"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 //go:embed asn1_ber.jq
 var asn1FS embed.FS
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.ASN1_BER,
 		Description: "ASN1 BER (basic encoding rules, also CER and DER)",
 		DecodeFn:    decodeASN1BER,

diff --git a/format/av1/av1_ccr.go b/format/av1/av1_ccr.go
@@ -6,13 +6,13 @@ package av1
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.AV1_CCR,
 		Description: "AV1 Codec Configuration Record",
 		DecodeFn:    ccrDecode,

diff --git a/format/av1/av1_frame.go b/format/av1/av1_frame.go
@@ -6,14 +6,14 @@ package av1
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 )
 
 var obuFormat decode.Group
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.AV1_FRAME,
 		Description: "AV1 frame",
 		DecodeFn:    frameDecode,

diff --git a/format/av1/av1_obu.go b/format/av1/av1_obu.go
@@ -2,13 +2,13 @@ package av1
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.AV1_OBU,
 		Description: "AV1 Open Bitstream Unit",
 		DecodeFn:    obuDecode,

diff --git a/format/avro/avro_ocf.go b/format/avro/avro_ocf.go
@@ -10,17 +10,17 @@ import (
 	"github.com/wader/fq/format"
 	"github.com/wader/fq/format/avro/decoders"
 	"github.com/wader/fq/format/avro/schema"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 //go:embed avro_ocf.jq
 var avroOcfFS embed.FS
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.AVRO_OCF,
 		Description: "Avro object container file",
 		Groups:      []string{format.PROBE},

diff --git a/format/bencode/bencode.go b/format/bencode/bencode.go
@@ -7,16 +7,16 @@ import (
 	"strconv"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 //go:embed bencode.jq
 var bencodeFS embed.FS
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.BENCODE,
 		Description: "BitTorrent bencoding",
 		DecodeFn:    decodeBencode,

diff --git a/format/bitcoin/bitcoin_blkdat.go b/format/bitcoin/bitcoin_blkdat.go
@@ -2,14 +2,14 @@ package bitcoin
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 )
 
 var bitcoinBlockFormat decode.Group
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.BITCOIN_BLKDAT,
 		Description: "Bitcoin blk.dat",
 		Groups:      []string{format.PROBE},

diff --git a/format/bitcoin/bitcoin_block.go b/format/bitcoin/bitcoin_block.go
@@ -6,15 +6,15 @@ import (
 	"fmt"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 var bitcoinTranscationFormat decode.Group
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.BITCOIN_BLOCK,
 		Description: "Bitcoin block",
 		Dependencies: []decode.Dependency{

diff --git a/format/bitcoin/bitcoin_script.go b/format/bitcoin/bitcoin_script.go
@@ -2,8 +2,8 @@ package bitcoin
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
@@ -34,7 +34,7 @@ func (ops opcodeEntries) MapScalar(s scalar.S) (scalar.S, error) {
 }
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.BITCOIN_SCRIPT,
 		Description: "Bitcoin script",
 		DecodeFn:    decodeBitcoinScript,

diff --git a/format/bitcoin/bitcoin_transaction.go b/format/bitcoin/bitcoin_transaction.go
@@ -6,15 +6,15 @@ package bitcoin
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 var bitcoinScriptFormat decode.Group
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.BITCOIN_TRANSACTION,
 		Description: "Bitcoin transaction",
 		Dependencies: []decode.Dependency{

diff --git a/format/bson/bson.go b/format/bson/bson.go
@@ -7,16 +7,16 @@ import (
 	"embed"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 //go:embed bson.jq
 var bsonFS embed.FS
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.BSON,
 		Description: "Binary JSON",
 		DecodeFn:    decodeBSON,

diff --git a/format/bzip2/bzip2.go b/format/bzip2/bzip2.go
@@ -13,16 +13,16 @@ import (
 	"math/bits"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 var probeGroup decode.Group
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.BZIP2,
 		Description: "bzip2 compression",
 		Groups:      []string{format.PROBE},

diff --git a/format/cbor/cbor.go b/format/cbor/cbor.go
@@ -13,18 +13,18 @@ import (
 	"strings"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 //go:embed cbor.jq
 var cborFS embed.FS
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.CBOR,
 		Description: "Concise Binary Object Representation",
 		DecodeFn:    decodeCBOR,

diff --git a/format/dns/dns.go b/format/dns/dns.go
@@ -8,13 +8,13 @@ import (
 	"strings"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.DNS,
 		Description: "DNS packet",
 		Groups: []string{

diff --git a/format/dns/dns_tcp.go b/format/dns/dns_tcp.go
@@ -2,12 +2,12 @@ package dns
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 )
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.DNS_TCP,
 		Description: "DNS packet (TCP)",
 		DecodeFn:    dnsTCPDecode,

diff --git a/format/elf/elf.go b/format/elf/elf.go
@@ -11,13 +11,13 @@ import (
 	"strings"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.ELF,
 		Description: "Executable and Linkable Format",
 		Groups:      []string{format.PROBE},

diff --git a/format/fairplay/fairplay.go b/format/fairplay/fairplay.go
@@ -4,12 +4,12 @@ package fairplay
 
 import (
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 )
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.FAIRPLAY_SPC,
 		Description: "FairPlay Server Playback Context",
 		DecodeFn:    fairPlaySPCDecode,

diff --git a/format/flac/flac.go b/format/flac/flac.go
@@ -12,18 +12,18 @@ import (
 	"fmt"
 
 	"github.com/wader/fq/format"
-	"github.com/wader/fq/format/registry"
 	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 var flacMetadatablocksFormat decode.Group
 var flacFrameFormat decode.Group
 
 func init() {
-	registry.MustRegister(decode.Format{
+	interp.RegisterFormat(decode.Format{
 		Name:        format.FLAC,
 		Description: "Free Lossless Audio Codec file",
 		Groups:      []string{format.PROBE},