decode: Support multiple format args and some rename and refactor

This will allow passing both cli options and format options to sub decoder. Ex: pass keylog option to a tls decoder when decoding a pcap. Ex: pass decode options to a format inside a http body inside a pcap. Add ArgAs method to lookup argument based on type. This also makes the format decode function have same signature as sub decoders in the decode API. This change decode.Format a bit: DecodeFn is now just func(d *D) any DecodeInArg renamed to DefaultInArg
wader · Feb 18, 2023 · 8e0dde0 · 8e0dde0
1 parent 570a213
commit 8e0dde0
Show file tree

Hide file tree

Showing 117 changed files with 299 additions and 249 deletions.
diff --git a/doc/presentations/bts2022/mp3.go b/doc/presentations/bts2022/mp3.go
@@ -1,4 +1,4 @@
-func decode(d *decode.D, _ any) any {
+func decode(d *decode.D) any {
 	d.FieldArray("headers", func(d *decode.D) {
 		for !d.End() {
 			d.TryFieldFormat("header", headerGroup)

diff --git a/format/ape/apev2.go b/format/ape/apev2.go
@@ -21,7 +21,7 @@ func init() {
 	})
 }
 
-func apev2Decode(d *decode.D, _ any) any {
+func apev2Decode(d *decode.D) any {
 	d.Endian = decode.LittleEndian
 
 	headerFooterFn := func(d *decode.D, name string) uint64 {

diff --git a/format/apple/bookmark/apple_bookmark.go b/format/apple/bookmark/apple_bookmark.go
@@ -379,7 +379,7 @@ const headerEndBitPos = headerEnd * 8
 // all offsets are calculated relative to the end of the bookmark header
 func calcOffset(i uint64) int64 { return int64(8 * (i + headerEnd)) }
 
-func bookmarkDecode(d *decode.D, _ any) any {
+func bookmarkDecode(d *decode.D) any {
 	// all fields are little-endian with the exception of the Date datatype.
 	d.Endian = decode.LittleEndian
 

diff --git a/format/apple/bplist/bplist.go b/format/apple/bplist/bplist.go
@@ -227,7 +227,7 @@ type plist struct {
 	pld      apple.PosLoopDetector[uint64]
 }
 
-func bplistDecode(d *decode.D, _ any) any {
+func bplistDecode(d *decode.D) any {
 	d.FieldStruct("header", func(d *decode.D) {
 		d.FieldUTF8("magic", 6, d.StrAssert("bplist"))
 		d.FieldUTF8("version", 2, d.StrAssert("00"))

diff --git a/format/apple/macho/macho.go b/format/apple/macho/macho.go
@@ -364,7 +364,7 @@ var sectionTypes = scalar.UintMapSymStr{
 	0x15: "thread_local_init_function_pointers",
 }
 
-func machoDecode(d *decode.D, _ any) any {
+func machoDecode(d *decode.D) any {
 	var archBits int
 	var cpuType uint64
 	var ncmds uint64

diff --git a/format/apple/macho/macho_fat.go b/format/apple/macho/macho_fat.go
@@ -26,7 +26,7 @@ func init() {
 //nolint:revive
 const FAT_MAGIC = 0xcafe_babe
 
-func machoFatDecode(d *decode.D, _ any) any {
+func machoFatDecode(d *decode.D) any {
 	type ofile struct {
 		offset int64
 		size   int64

diff --git a/format/ar/ar.go b/format/ar/ar.go
@@ -21,7 +21,7 @@ func init() {
 	})
 }
 
-func decodeAr(d *decode.D, _ any) any {
+func decodeAr(d *decode.D) any {
 	d.FieldUTF8("signature", 8, d.StrAssert("!<arch>\n"))
 	d.FieldArray("files", func(d *decode.D) {
 		for !d.End() {

diff --git a/format/asn1/asn1_ber.go b/format/asn1/asn1_ber.go
@@ -413,7 +413,7 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
 	})
 }
 
-func decodeASN1BER(d *decode.D, _ any) any {
+func decodeASN1BER(d *decode.D) any {
 	decodeASN1BERValue(d, nil, nil, formConstructed, universalTypeSequence)
 	return nil
 }
diff --git a/format/av1/av1_ccr.go b/format/av1/av1_ccr.go
@@ -19,7 +19,7 @@ func init() {
 	})
 }
 
-func ccrDecode(d *decode.D, _ any) any {
+func ccrDecode(d *decode.D) any {
 	d.FieldU1("marker")
 	d.FieldU7("version")
 	d.FieldU3("seq_profile")

diff --git a/format/av1/av1_frame.go b/format/av1/av1_frame.go
@@ -25,7 +25,7 @@ func init() {
 	})
 }
 
-func frameDecode(d *decode.D, _ any) any {
+func frameDecode(d *decode.D) any {
 	for d.NotEnd() {
 		d.FieldFormat("obu", obuFormat, nil)
 	}

diff --git a/format/av1/av1_obu.go b/format/av1/av1_obu.go
@@ -40,7 +40,7 @@ var obuTypeNames = scalar.UintMapSymStr{
 	OBU_PADDING:                "OBU_PADDING",
 }
 
-func obuDecode(d *decode.D, _ any) any {
+func obuDecode(d *decode.D) any {
 	var obuType uint64
 	var obuSize int64
 	hasExtension := false

diff --git a/format/avro/avro_ocf.go b/format/avro/avro_ocf.go
@@ -128,7 +128,7 @@ func decodeBlockCodec(d *decode.D, dataSize int64, codec string) *bytes.Buffer {
 	return bb
 }
 
-func decodeAvroOCF(d *decode.D, _ any) any {
+func decodeAvroOCF(d *decode.D) any {
 	header := decodeHeader(d)
 
 	decodeFn, err := decoders.DecodeFnForSchema(header.Schema)

diff --git a/format/bencode/bencode.go b/format/bencode/bencode.go
@@ -91,7 +91,7 @@ func decodeBencodeValue(d *decode.D) {
 	}
 }
 
-func decodeBencode(d *decode.D, _ any) any {
+func decodeBencode(d *decode.D) any {
 	decodeBencodeValue(d)
 	return nil
 }
diff --git a/format/bitcoin/bitcoin_blkdat.go b/format/bitcoin/bitcoin_blkdat.go
@@ -22,7 +22,7 @@ func init() {
 	})
 }
 
-func decodeBlkDat(d *decode.D, in any) any {
+func decodeBlkDat(d *decode.D) any {
 	validBlocks := 0
 	for !d.End() {
 		d.FieldFormat("block", bitcoinBlockFormat, format.BitCoinBlockIn{HasHeader: true})

diff --git a/format/bitcoin/bitcoin_block.go b/format/bitcoin/bitcoin_block.go
@@ -22,7 +22,7 @@ func init() {
 			{Names: []string{format.BITCOIN_TRANSACTION}, Group: &bitcoinTranscationFormat},
 		},
 		DecodeFn: decodeBitcoinBlock,
-		DecodeInArg: format.BitCoinBlockIn{
+		DefaultInArg: format.BitCoinBlockIn{
 			HasHeader: false,
 		},
 	})
@@ -35,8 +35,10 @@ var rawHexReverse = scalar.BitBufFn(func(s scalar.BitBuf) (scalar.BitBuf, error)
 	})
 })
 
-func decodeBitcoinBlock(d *decode.D, in any) any {
-	bbi, _ := in.(format.BitCoinBlockIn)
+func decodeBitcoinBlock(d *decode.D) any {
+	var bbi format.BitCoinBlockIn
+	d.ArgAs(&bbi)
+
 	size := d.BitsLeft()
 
 	if bbi.HasHeader {

diff --git a/format/bitcoin/bitcoin_script.go b/format/bitcoin/bitcoin_script.go
@@ -43,7 +43,7 @@ func init() {
 	})
 }
 
-func decodeBitcoinScript(d *decode.D, in any) any {
+func decodeBitcoinScript(d *decode.D) any {
 	// based on https://en.bitcoin.it/wiki/Script
 	opcodeEntries := opcodeEntries{
 		{r: [2]byte{0x00, 0x00}, s: scalar.Uint{Sym: "false"}},

diff --git a/format/bitcoin/bitcoin_transaction.go b/format/bitcoin/bitcoin_transaction.go
@@ -44,7 +44,7 @@ func decodeVarInt(d *decode.D) uint64 {
 // all zero
 var txIDCoinbaseBytes = [32]byte{}
 
-func decodeBitcoinTranscation(d *decode.D, in any) any {
+func decodeBitcoinTranscation(d *decode.D) any {
 	d.Endian = decode.LittleEndian
 
 	d.FieldU32("version")

diff --git a/format/bits/bits.go b/format/bits/bits.go
@@ -13,8 +13,8 @@ import (
 //go:embed bytes.md
 var bitsFS embed.FS
 
-func decodeBits(unit int) func(d *decode.D, _ any) any {
-	return func(d *decode.D, _ any) any {
+func decodeBits(unit int) func(d *decode.D) any {
+	return func(d *decode.D) any {
 		var s scalar.Any
 		b, _ := interp.NewBinaryFromBitReader(d.BitBufRange(0, d.Len()), unit, 0)
 		s.Actual = b

diff --git a/format/bson/bson.go b/format/bson/bson.go
@@ -111,7 +111,7 @@ func decodeBSONDocument(d *decode.D) {
 	})
 }
 
-func decodeBSON(d *decode.D, _ any) any {
+func decodeBSON(d *decode.D) any {
 	d.Endian = decode.LittleEndian
 
 	decodeBSONDocument(d)

diff --git a/format/bzip2/bzip2.go b/format/bzip2/bzip2.go
@@ -48,7 +48,7 @@ func (bfr bitFlipReader) Read(p []byte) (n int, err error) {
 	return n, err
 }
 
-func bzip2Decode(d *decode.D, _ any) any {
+func bzip2Decode(d *decode.D) any {
 	// moreStreams := true
 
 	// d.FieldArray("streams", func(d *decode.D) {

diff --git a/format/cbor/cbor.go b/format/cbor/cbor.go
@@ -263,7 +263,7 @@ func decodeCBORValue(d *decode.D) any {
 	panic("unreachable")
 }
 
-func decodeCBOR(d *decode.D, _ any) any {
+func decodeCBOR(d *decode.D) any {
 	decodeCBORValue(d)
 	return nil
 }
diff --git a/format/csv/csv.go b/format/csv/csv.go
@@ -26,7 +26,7 @@ func init() {
 		Description: "Comma separated values",
 		ProbeOrder:  format.ProbeOrderTextFuzzy,
 		DecodeFn:    decodeCSV,
-		DecodeInArg: format.CSVLIn{
+		DefaultInArg: format.CSVLIn{
 			Comma:   ",",
 			Comment: "#",
 		},
@@ -36,8 +36,9 @@ func init() {
 	interp.RegisterFunc1("_to_csv", toCSV)
 }
 
-func decodeCSV(d *decode.D, in any) any {
-	ci, _ := in.(format.CSVLIn)
+func decodeCSV(d *decode.D) any {
+	var ci format.CSVLIn
+	d.ArgAs(&ci)
 
 	var rvs []any
 	br := d.RawLen(d.Len())

diff --git a/format/dns/dns.go b/format/dns/dns.go
@@ -261,9 +261,11 @@ func dnsDecode(d *decode.D, hasLengthHeader bool) any {
 	return nil
 }
 
-func dnsUDPDecode(d *decode.D, in any) any {
-	if upi, ok := in.(format.UDPPayloadIn); ok {
+func dnsUDPDecode(d *decode.D) any {
+	var upi format.UDPPayloadIn
+	if d.ArgAs(&upi) {
 		upi.MustIsPort(d.Fatalf, format.UDPPortDomain, format.UDPPortMDNS)
 	}
+
 	return dnsDecode(d, false)
 }
diff --git a/format/dns/dns_tcp.go b/format/dns/dns_tcp.go
@@ -15,8 +15,9 @@ func init() {
 	})
 }
 
-func dnsTCPDecode(d *decode.D, in any) any {
-	if tsi, ok := in.(format.TCPStreamIn); ok {
+func dnsTCPDecode(d *decode.D) any {
+	var tsi format.TCPStreamIn
+	if d.ArgAs(&tsi) {
 		tsi.MustIsPort(d.Fatalf, format.TCPPortDomain)
 	}
 	return dnsDecode(d, true)

diff --git a/format/elf/elf.go b/format/elf/elf.go
@@ -1100,7 +1100,7 @@ func elfDecodeSectionHeaders(d *decode.D, ec elfContext) {
 	}
 }
 
-func elfDecode(d *decode.D, _ any) any {
+func elfDecode(d *decode.D) any {
 	var ec elfContext
 
 	d.FieldStruct("header", func(d *decode.D) { elfDecodeHeader(d, &ec) })

diff --git a/format/fairplay/fairplay.go b/format/fairplay/fairplay.go
@@ -16,7 +16,7 @@ func init() {
 	})
 }
 
-func fairPlaySPCDecode(d *decode.D, _ any) any {
+func fairPlaySPCDecode(d *decode.D) any {
 	d.FieldU32("version")
 	d.FieldRawLen("reserved", 32)
 	d.FieldRawLen("iv", 16*8)

diff --git a/format/flac/flac.go b/format/flac/flac.go
@@ -32,7 +32,7 @@ func init() {
 	})
 }
 
-func flacDecode(d *decode.D, _ any) any {
+func flacDecode(d *decode.D) any {
 	d.FieldUTF8("magic", 4, d.StrAssert("fLaC"))
 
 	var streamInfo format.FlacStreamInfo

diff --git a/format/flac/flac_frame.go b/format/flac/flac_frame.go
@@ -17,7 +17,7 @@ func init() {
 		Name:        format.FLAC_FRAME,
 		Description: "FLAC frame",
 		DecodeFn:    frameDecode,
-		DecodeInArg: format.FlacFrameIn{
+		DefaultInArg: format.FlacFrameIn{
 			BitsPerSample: 16, // TODO: maybe should not have a default value?
 		},
 	})
@@ -100,16 +100,16 @@ func utf8Uint(d *decode.D) uint64 {
 }
 
 // in argument is an optional FlacFrameIn struct with stream info
-func frameDecode(d *decode.D, in any) any {
+func frameDecode(d *decode.D) any {
 	frameStart := d.Pos()
 	blockSize := 0
 	channelAssignment := uint64(0)
 	channels := 0
 	sampleSize := 0
 	sideChannelIndex := -1
 
-	ffi, ok := in.(format.FlacFrameIn)
-	if ok {
+	var ffi format.FlacFrameIn
+	if d.ArgAs(&ffi) {
 		sampleSize = ffi.BitsPerSample
 	}
 

diff --git a/format/flac/flac_metadatablock.go b/format/flac/flac_metadatablock.go
@@ -49,7 +49,7 @@ var metadataBlockNames = scalar.UintMapSymStr{
 	MetadataBlockPicture:       "picture",
 }
 
-func metadatablockDecode(d *decode.D, _ any) any {
+func metadatablockDecode(d *decode.D) any {
 	var hasStreamInfo bool
 	var streamInfo format.FlacStreamInfo
 

diff --git a/format/flac/flac_metadatablocks.go b/format/flac/flac_metadatablocks.go
@@ -23,7 +23,7 @@ func init() {
 	})
 }
 
-func metadatablocksDecode(d *decode.D, _ any) any {
+func metadatablocksDecode(d *decode.D) any {
 	flacMetadatablocksOut := format.FlacMetadatablocksOut{}
 
 	isLastBlock := false

diff --git a/format/flac/flac_picture.go b/format/flac/flac_picture.go
@@ -44,7 +44,7 @@ func init() {
 	})
 }
 
-func pictureDecode(d *decode.D, _ any) any {
+func pictureDecode(d *decode.D) any {
 	lenStr := func(name string) {
 		l := d.FieldU32(name + "_length")
 		d.FieldUTF8(name, int(l))

diff --git a/format/flac/flac_streaminfo.go b/format/flac/flac_streaminfo.go
@@ -15,7 +15,7 @@ func init() {
 	})
 }
 
-func streaminfoDecode(d *decode.D, _ any) any {
+func streaminfoDecode(d *decode.D) any {
 	d.FieldU16("minimum_block_size")
 	d.FieldU16("maximum_block_size")
 	d.FieldU24("minimum_frame_size")

diff --git a/format/flv/flv.go b/format/flv/flv.go
@@ -65,7 +65,7 @@ var typeNames = scalar.UintMapSymStr{
 	typeLongString:  "LongString",
 }
 
-func flvDecode(d *decode.D, _ any) any {
+func flvDecode(d *decode.D) any {
 	var fieldScriptDataObject func()
 	var fieldScriptDataVariable func(d *decode.D, name string)
 

diff --git a/format/gif/gif.go b/format/gif/gif.go
@@ -51,7 +51,7 @@ func fieldColorMap(d *decode.D, name string, bitDepth int) {
 	})
 }
 
-func gifDecode(d *decode.D, _ any) any {
+func gifDecode(d *decode.D) any {
 	d.Endian = decode.LittleEndian
 
 	d.FieldUTF8("header", 6, d.StrAssert("GIF87a", "GIF89a"))

diff --git a/format/gzip/gzip.go b/format/gzip/gzip.go
@@ -58,7 +58,7 @@ var deflateExtraFlagsNames = scalar.UintMapSymStr{
 	4: "fast",
 }
 
-func gzDecode(d *decode.D, _ any) any {
+func gzDecode(d *decode.D) any {
 	d.Endian = decode.LittleEndian
 
 	d.FieldRawLen("identification", 2*8, d.AssertBitBuf([]byte("\x1f\x8b")))

diff --git a/format/icc/icc_profile.go b/format/icc/icc_profile.go
@@ -81,7 +81,7 @@ func decodeBCDU8(d *decode.D) uint64 {
 	return (n>>4)*10 + n&0xf
 }
 
-func iccProfileDecode(d *decode.D, _ any) any {
+func iccProfileDecode(d *decode.D) any {
 	/*
 	   0..3 Profile size uInt32Number
 	   4..7 CMM Type signature see below

diff --git a/format/id3/id3v1.go b/format/id3/id3v1.go
@@ -18,7 +18,7 @@ func init() {
 }
 
 // Decode ID3v1 tag
-func id3v1Decode(d *decode.D, _ any) any {
+func id3v1Decode(d *decode.D) any {
 	d.AssertAtLeastBitsLeft(128 * 8)
 	d.FieldUTF8("magic", 3, d.StrAssert("TAG"))
 	if d.PeekBits(8) == uint64('+') {

diff --git a/format/id3/id3v11.go b/format/id3/id3v11.go
@@ -15,7 +15,7 @@ func init() {
 	})
 }
 
-func id3v11Decode(d *decode.D, _ any) any {
+func id3v11Decode(d *decode.D) any {
 	d.AssertAtLeastBitsLeft(128 * 8)
 	d.FieldUTF8("magic", 4, d.StrAssert("TAG+"))
 	d.FieldUTF8("title", 60)