decode,interp: Refactor format groups into a proper struct

Replaces []Format with a Group type. A bit more type safe. Breaking change for RegisterFormat, now takes a first argument that is a "single" format group. Lots of naming cleanup. This is also preparation for decode group argument which will enable doing intresting probing, ex a format decoder could know it's decode as part of probe group (html could be probed possibly), or have "arg probe" group for decoder who inspect args to know if they should probe (-d /path/to/schema etc) to enable nice CLI-ergonomics.
wader · Apr 29, 2023 · b08ef00 · b08ef00
1 parent b2cb5c3
commit b08ef00
Show file tree

Hide file tree

Showing 133 changed files with 1,621 additions and 1,508 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -10,12 +10,7 @@
             "program": ".",
             "cwd": "${workspaceFolder}",
             "env": {},
-            "args": [
-                "-d",
-                "mp4",
-                ".",
-                "file"
-            ],
+            "args": [".", "doc/file.mp3"]
         }
     ]
-}
+}
diff --git a/format/ape/apev2.go b/format/ape/apev2.go
@@ -8,17 +8,18 @@ import (
 	"github.com/wader/fq/pkg/interp"
 )
 
-var imageFormat decode.Group
+var imageGroup decode.Group
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.APEV2,
-		Description: "APEv2 metadata tag",
-		DecodeFn:    apev2Decode,
-		Dependencies: []decode.Dependency{
-			{Names: []string{format.IMAGE}, Group: &imageFormat},
-		},
-	})
+	interp.RegisterFormat(
+		format.Apev2,
+		&decode.Format{
+			Description: "APEv2 metadata tag",
+			DecodeFn:    apev2Decode,
+			Dependencies: []decode.Dependency{
+				{Groups: []*decode.Group{format.Image}, Out: &imageGroup},
+			},
+		})
 }
 
 func apev2Decode(d *decode.D) any {
@@ -59,7 +60,7 @@ func apev2Decode(d *decode.D) any {
 					d.FramedFn(int64(itemSize)*8, func(d *decode.D) {
 						d.FieldUTF8Null("filename")
 						// assume image if binary
-						d.FieldFormatOrRaw("value", imageFormat, nil)
+						d.FieldFormatOrRaw("value", &imageGroup, nil)
 					})
 				} else {
 					d.FieldUTF8("value", int(itemSize))

diff --git a/format/apple/bookmark/apple_bookmark.go b/format/apple/bookmark/apple_bookmark.go
@@ -15,14 +15,14 @@ import (
 var bookmarkFS embed.FS
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.APPLE_BOOKMARK,
-		ProbeOrder:  format.ProbeOrderBinUnique,
-		Description: "Apple BookmarkData",
-		Groups:      []string{format.PROBE},
-		DecodeFn:    bookmarkDecode,
-		Functions:   []string{"torepr"},
-	})
+	interp.RegisterFormat(format.AppleBookmark,
+		&decode.Format{
+			ProbeOrder:  format.ProbeOrderBinUnique,
+			Description: "Apple BookmarkData",
+			Groups:      []*decode.Group{format.Probe},
+			DecodeFn:    bookmarkDecode,
+			Functions:   []string{"torepr"},
+		})
 	interp.RegisterFS(bookmarkFS)
 }
 

diff --git a/format/apple/bplist/bplist.go b/format/apple/bplist/bplist.go
@@ -17,14 +17,15 @@ import (
 var bplistFS embed.FS
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.BPLIST,
-		ProbeOrder:  format.ProbeOrderBinUnique,
-		Description: "Apple Binary Property List",
-		Groups:      []string{format.PROBE},
-		DecodeFn:    bplistDecode,
-		Functions:   []string{"torepr"},
-	})
+	interp.RegisterFormat(
+		format.Bplist,
+		&decode.Format{
+			ProbeOrder:  format.ProbeOrderBinUnique,
+			Description: "Apple Binary Property List",
+			Groups:      []*decode.Group{format.Probe},
+			DecodeFn:    bplistDecode,
+			Functions:   []string{"torepr"},
+		})
 	interp.RegisterFS(bplistFS)
 }
 

diff --git a/format/apple/macho/macho.go b/format/apple/macho/macho.go
@@ -18,12 +18,13 @@ import (
 var machoFS embed.FS
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.MACHO,
-		Description: "Mach-O macOS executable",
-		Groups:      []string{format.PROBE},
-		DecodeFn:    machoDecode,
-	})
+	interp.RegisterFormat(
+		format.Macho,
+		&decode.Format{
+			Description: "Mach-O macOS executable",
+			Groups:      []*decode.Group{format.Probe},
+			DecodeFn:    machoDecode,
+		})
 	interp.RegisterFS(machoFS)
 }
 

diff --git a/format/apple/macho/macho_fat.go b/format/apple/macho/macho_fat.go
@@ -12,15 +12,16 @@ import (
 var machoFormat decode.Group
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.MACHO_FAT,
-		Description: "Fat Mach-O macOS executable (multi-architecture)",
-		Groups:      []string{format.PROBE},
-		DecodeFn:    machoFatDecode,
-		Dependencies: []decode.Dependency{
-			{Names: []string{format.MACHO}, Group: &machoFormat},
-		},
-	})
+	interp.RegisterFormat(
+		format.MachoFat,
+		&decode.Format{
+			Description: "Fat Mach-O macOS executable (multi-architecture)",
+			Groups:      []*decode.Group{format.Probe},
+			DecodeFn:    machoFatDecode,
+			Dependencies: []decode.Dependency{
+				{Groups: []*decode.Group{format.Macho}, Out: &machoFormat},
+			},
+		})
 }
 
 const FAT_MAGIC = 0xcafe_babe
@@ -55,7 +56,7 @@ func machoFatDecode(d *decode.D) any {
 	d.FieldArray("files", func(d *decode.D) {
 		for _, o := range ofiles {
 			d.RangeFn(o.offset*8, o.size*8, func(d *decode.D) {
-				d.FieldFormat("file", machoFormat, nil)
+				d.FieldFormat("file", &machoFormat, nil)
 			})
 		}
 	})

diff --git a/format/ar/ar.go b/format/ar/ar.go
@@ -7,18 +7,19 @@ import (
 	"github.com/wader/fq/pkg/scalar"
 )
 
-var probeFormat decode.Group
+var probeGroup decode.Group
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.AR,
-		Description: "Unix archive",
-		Groups:      []string{format.PROBE},
-		DecodeFn:    decodeAr,
-		Dependencies: []decode.Dependency{
-			{Names: []string{format.PROBE}, Group: &probeFormat},
-		},
-	})
+	interp.RegisterFormat(
+		format.Ar,
+		&decode.Format{
+			Description: "Unix archive",
+			Groups:      []*decode.Group{format.Probe},
+			DecodeFn:    decodeAr,
+			Dependencies: []decode.Dependency{
+				{Groups: []*decode.Group{format.Probe}, Out: &probeGroup},
+			},
+		})
 }
 
 func decodeAr(d *decode.D) any {
@@ -37,7 +38,7 @@ func decodeAr(d *decode.D) any {
 				}
 				size := int64(sizeStr.SymUint()) * 8
 				d.FieldUTF8("ending_characters", 2)
-				d.FieldFormatOrRawLen("data", size, probeFormat, nil)
+				d.FieldFormatOrRawLen("data", size, &probeGroup, nil)
 				padding := d.AlignBits(16)
 				if padding > 0 {
 					d.FieldRawLen("padding", int64(padding))

diff --git a/format/asn1/asn1_ber.go b/format/asn1/asn1_ber.go
@@ -33,12 +33,13 @@ import (
 var asn1FS embed.FS
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.ASN1_BER,
-		Description: "ASN1 BER (basic encoding rules, also CER and DER)",
-		DecodeFn:    decodeASN1BER,
-		Functions:   []string{"torepr"},
-	})
+	interp.RegisterFormat(
+		format.Asn1Ber,
+		&decode.Format{
+			Description: "ASN1 BER (basic encoding rules, also CER and DER)",
+			DecodeFn:    decodeASN1BER,
+			Functions:   []string{"torepr"},
+		})
 	interp.RegisterFS(asn1FS)
 }
 

diff --git a/format/av1/av1_ccr.go b/format/av1/av1_ccr.go
@@ -12,11 +12,12 @@ import (
 )
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.AV1_CCR,
-		Description: "AV1 Codec Configuration Record",
-		DecodeFn:    ccrDecode,
-	})
+	interp.RegisterFormat(
+		format.Av1Ccr,
+		&decode.Format{
+			Description: "AV1 Codec Configuration Record",
+			DecodeFn:    ccrDecode,
+		})
 }
 
 func ccrDecode(d *decode.D) any {

diff --git a/format/av1/av1_frame.go b/format/av1/av1_frame.go
@@ -10,24 +10,25 @@ import (
 	"github.com/wader/fq/pkg/interp"
 )
 
-var obuFormat decode.Group
+var av1FrameObuGroup decode.Group
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.AV1_FRAME,
-		Description: "AV1 frame",
-		DecodeFn:    frameDecode,
-		RootArray:   true,
-		RootName:    "frame",
-		Dependencies: []decode.Dependency{
-			{Names: []string{format.AV1_OBU}, Group: &obuFormat},
-		},
-	})
+	interp.RegisterFormat(
+		format.Av1Frame,
+		&decode.Format{
+			Description: "AV1 frame",
+			DecodeFn:    frameDecode,
+			RootArray:   true,
+			RootName:    "frame",
+			Dependencies: []decode.Dependency{
+				{Groups: []*decode.Group{format.Av1Obu}, Out: &av1FrameObuGroup},
+			},
+		})
 }
 
 func frameDecode(d *decode.D) any {
 	for d.NotEnd() {
-		d.FieldFormat("obu", obuFormat, nil)
+		d.FieldFormat("obu", &av1FrameObuGroup, nil)
 	}
 
 	return nil

diff --git a/format/av1/av1_obu.go b/format/av1/av1_obu.go
@@ -8,11 +8,12 @@ import (
 )
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.AV1_OBU,
-		Description: "AV1 Open Bitstream Unit",
-		DecodeFn:    obuDecode,
-	})
+	interp.RegisterFormat(
+		format.Av1Obu,
+		&decode.Format{
+			Description: "AV1 Open Bitstream Unit",
+			DecodeFn:    obuDecode,
+		})
 }
 
 const (

diff --git a/format/avro/avro_ocf.go b/format/avro/avro_ocf.go
@@ -20,12 +20,13 @@ import (
 var avroOcfFS embed.FS
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.AVRO_OCF,
-		Description: "Avro object container file",
-		Groups:      []string{format.PROBE},
-		DecodeFn:    decodeAvroOCF,
-	})
+	interp.RegisterFormat(
+		format.AvroOcf,
+		&decode.Format{
+			Description: "Avro object container file",
+			Groups:      []*decode.Group{format.Probe},
+			DecodeFn:    decodeAvroOCF,
+		})
 	interp.RegisterFS(avroOcfFS)
 }
 

diff --git a/format/bencode/bencode.go b/format/bencode/bencode.go
@@ -17,12 +17,13 @@ import (
 var bencodeFS embed.FS
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.BENCODE,
-		Description: "BitTorrent bencoding",
-		DecodeFn:    decodeBencode,
-		Functions:   []string{"torepr"},
-	})
+	interp.RegisterFormat(
+		format.Bencode,
+		&decode.Format{
+			Description: "BitTorrent bencoding",
+			DecodeFn:    decodeBencode,
+			Functions:   []string{"torepr"},
+		})
 	interp.RegisterFS(bencodeFS)
 }
 

diff --git a/format/bitcoin/bitcoin_blkdat.go b/format/bitcoin/bitcoin_blkdat.go
@@ -6,26 +6,27 @@ import (
 	"github.com/wader/fq/pkg/interp"
 )
 
-var bitcoinBlockFormat decode.Group
+var bitcoinBlockGroup decode.Group
 
 func init() {
-	interp.RegisterFormat(decode.Format{
-		Name:        format.BITCOIN_BLKDAT,
-		Description: "Bitcoin blk.dat",
-		Groups:      []string{format.PROBE},
-		Dependencies: []decode.Dependency{
-			{Names: []string{format.BITCOIN_BLOCK}, Group: &bitcoinBlockFormat},
-		},
-		DecodeFn:  decodeBlkDat,
-		RootArray: true,
-		RootName:  "blocks",
-	})
+	interp.RegisterFormat(
+		format.BitcoinBlkdat,
+		&decode.Format{
+			Description: "Bitcoin blk.dat",
+			Groups:      []*decode.Group{format.Probe},
+			Dependencies: []decode.Dependency{
+				{Groups: []*decode.Group{format.BitcoinBlock}, Out: &bitcoinBlockGroup},
+			},
+			DecodeFn:  decodeBlkDat,
+			RootArray: true,
+			RootName:  "blocks",
+		})
 }
 
 func decodeBlkDat(d *decode.D) any {
 	validBlocks := 0
 	for !d.End() {
-		d.FieldFormat("block", bitcoinBlockFormat, format.BitCoinBlockIn{HasHeader: true})
+		d.FieldFormat("block", &bitcoinBlockGroup, format.BitCoinBlockIn{HasHeader: true})
 		validBlocks++
 	}