[WIP] initial attempt to add postgres

format/all/all.go

@@ -24,6 +24,7 @@ import (
 	_ "github.com/wader/fq/format/opus"
 	_ "github.com/wader/fq/format/pcap"
 	_ "github.com/wader/fq/format/png"
+	_ "github.com/wader/fq/format/postgres"
 	_ "github.com/wader/fq/format/protobuf"
 	_ "github.com/wader/fq/format/raw"
 	_ "github.com/wader/fq/format/tar"

format/format.go

@@ -73,6 +73,10 @@ const (
 	OPUS_PACKET         = "opus_packet"
 	PCAP                = "pcap"
 	PCAPNG              = "pcapng"
+	PGWAL               = "pgwal"
+	PGWALPAGE           = "pgwal_page"
+	PGMULTIXACTOFF      = "pgmultixact_offsets"
+	PGMULTIXACTMEM      = "pgmultixact_members"
 	PNG                 = "png"
 	PROTOBUF            = "protobuf"
 	PROTOBUF_WIDEVINE   = "protobuf_widevine"

format/postgres/pgmultixact.go

@@ -0,0 +1,86 @@
+package postgres
+
+import (
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/format/registry"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+const BLCKSZ = 8192
+
+func init() {
+	registry.MustRegister(decode.Format{
+		Name:        format.PGMULTIXACTOFF,
+		Description: "PostgreSQL multixact offset file",
+		DecodeFn:    mxOffsetDecode,
+	})
+	registry.MustRegister(decode.Format{
+		Name:        format.PGMULTIXACTMEM,
+		Description: "PostgreSQL multixact members file",
+		DecodeFn:    mxMembersDecode,
+	})
+}
+
+func mxOffsetDecode(d *decode.D, in interface{}) interface{} {
+	d.Endian = decode.LittleEndian
+
+	d.FieldArray("offsets", func(d *decode.D) {
+		for {
+			if d.End() {
+				break
+			}
+			d.FieldU32("offset", scalar.Hex)
+
+		}
+	})
+	return nil
+}
+
+var flags = scalar.UToScalar{
+	0: {Sym: "ForKeyShare", Description: "For Key Share"},
+	1: {Sym: "ForShare", Description: "For Share"},
+	2: {Sym: "ForNoKeyUpdate", Description: "For No Key Update"},
+	3: {Sym: "ForUpdate", Description: "For Update"},
+	4: {Sym: "NoKeyUpdate", Description: "No Key Update"},
+	5: {Sym: "Update", Description: "Update"},
+}
+
+func mxMembersDecode(d *decode.D, in interface{}) interface{} {
+	var xidLen uint = 4
+	var groupLen uint = 4 * (1 + xidLen)
+	d.Endian = decode.LittleEndian
+
+	m := d.FieldArrayValue("members")
+	p := d.FieldArrayValue("paddings")
+
+	for {
+		var xacts []*decode.D = make([]*decode.D, 4)
+
+		for i := 0; i < 4; i++ {
+			xacts[i] = m.FieldStructValue("xact")
+			xacts[i].FieldU8("status", flags)
+		}
+
+		for i := 0; i < 4; i++ {
+			xacts[i].FieldU32("xid")
+		}
+
+		// Check if rest of bytes are padding before EOF
+		if d.BitsLeft() < int64(groupLen*8) && d.BitsLeft() > 0 {
+			p.FieldRawLen("padding", d.BitsLeft())
+			break
+		}
+
+		// Check on EOF
+		if d.End() {
+			break
+		}
+
+		// Not EOF, let's check on block boundary
+		if blkLeft := BLCKSZ - (uint(d.Pos())>>3)%BLCKSZ; blkLeft < groupLen {
+			p.FieldRawLen("padding", int64(blkLeft*8))
+		}
+	}
+	return nil
+}

format/postgres/pgwal.go

@@ -0,0 +1,180 @@
+package postgres
+
+import (
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/format/registry"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+func init() {
+	registry.MustRegister(decode.Format{
+		Name:        format.PGWAL,
+		Description: "PostgreSQL write-ahead log file",
+		DecodeFn:    pgwalDecode,
+	})
+}
+
+const XLOG_BLCKSZ = 8192
+
+const XLP_LONG_HEADER = 2
+
+const (
+	BKPBLOCK_FORK_MASK = 0x0F
+	/* block data is an XLogRecordBlockImage */
+	BKPBLOCK_HAS_IMAGE = 0x10
+	BKPBLOCK_HAS_DATA  = 0x20
+	/* redo will re-init the page */
+	BKPBLOCK_WILL_INIT = 0x40
+	/* RelFileNode omitted, same as previous */
+	BKPBLOCK_SAME_REL = 0x80
+)
+
+/* Information stored in bimg_info */
+const (
+	/* page image has "hole" */
+	BKPIMAGE_HAS_HOLE = 0x01
+	/* page image is compressed */
+	BKPIMAGE_IS_COMPRESSED = 0x02
+	/* page image should be restored during replay */
+	BKPIMAGE_APPLY = 0x04
+)
+
+var expected_rem_len uint64 = 0
+
+var rmgrIds = scalar.UToScalar{
+	0:  {Sym: "XLOG", Description: "RM_XLOG_ID"},
+	1:  {Sym: "Transaction", Description: "RM_XACT_ID"},
+	2:  {Sym: "Storage", Description: "RM_SMGR_ID"},
+	3:  {Sym: "CLOG", Description: "RM_CLOG_ID"},
+	4:  {Sym: "Database", Description: "RM_DBASE_ID"},
+	5:  {Sym: "Tablespace", Description: "RM_TBLSPC_ID"},
+	6:  {Sym: "MultiXact", Description: "RM_MULTIXACT_ID"},
+	7:  {Sym: "RelMap", Description: "RM_RELMAP_ID"},
+	8:  {Sym: "Standby", Description: "RM_STANDBY_ID"},
+	9:  {Sym: "Heap2", Description: "RM_HEAP2_ID"},
+	10: {Sym: "Heap", Description: "RM_HEAP_ID"},
+	11: {Sym: "Btree", Description: "RM_BTREE_ID"},
+	12: {Sym: "Hash", Description: "RM_HASH_ID"},
+	13: {Sym: "Gin", Description: "RM_GIN_ID"},
+	14: {Sym: "Gist", Description: "RM_GIST_ID"},
+	15: {Sym: "Sequence", Description: "RM_SEQ_ID"},
+	16: {Sym: "SPGist", Description: "RM_SPGIST_ID"},
+	17: {Sym: "BRIN", Description: "RM_BRIN_ID"},
+	18: {Sym: "CommitTs", Description: "RM_COMMIT_TS_ID"},
+	19: {Sym: "ReplicationOrigin", Description: "RM_REPLORIGIN_ID"},
+	20: {Sym: "Generic", Description: "RM_GENERIC_ID"},
+	21: {Sym: "LogicalMessage", Description: "RM_LOGICALMSG_ID"},
+}
+
+func decodeXLogPageHeaderData(d *decode.D) {
+	var info uint64
+
+	d.FieldU16("xlp_magic", d.AssertU(0xd106))
+	info = d.FieldU16("xlp_info")
+	d.FieldU32("xlp_timeline")
+	d.FieldU64("xlp_pageaddr")
+	d.FieldU32("xlp_rem_len")
+	d.FieldRawLen("padding", int64(d.AlignBits(64)))
+	if info&XLP_LONG_HEADER != 0 {
+		// Long header
+		d.FieldStruct("XLogLongPageHeaderData", func(d *decode.D) {
+			d.FieldU64("xlp_sysid")
+			d.FieldU32("xlp_seg_size")
+			d.FieldU32("xlp_xlog_blcksz")
+		})
+	}
+}
+
+func pgwalDecode(d *decode.D, in interface{}) interface{} {
+
+	d.Endian = decode.LittleEndian
+
+	pageHeaders := d.FieldArrayValue("XLogPageHeaders")
+	header := pageHeaders.FieldStruct("XLogPageHeaderData", decodeXLogPageHeaderData)
+
+	d.FieldRawLen("prev_file_rec", int64(header.FieldGet("xlp_rem_len").V.(uint32)*8))
+	d.FieldRawLen("prev_file_rec_padding", int64(d.AlignBits(64)))
+
+	d.FieldArray("XLogRecords", func(d *decode.D) {
+		for {
+			d.FieldStruct("XLogRecord", func(d *decode.D) {
+				record_pos := uint64(d.Pos()) >> 3
+				record_len := d.FieldU32("xl_tot_len")
+				record_end := record_pos + record_len
+				header_pos := record_end - record_end%XLOG_BLCKSZ
+				d.FieldU32("xl_xid")
+				d.FieldU64("xl_prev", scalar.Hex)
+				d.FieldU8("xl_info")
+				d.FieldU8("xl_rmid", rmgrIds)
+				d.FieldRawLen("padding", int64(d.AlignBits(32)))
+				d.FieldU32("xl_crc", scalar.Hex)
+
+				var lenghts []uint64 = []uint64{}
+
+				d.FieldArray("XLogRecordBlockHeader", func(d *decode.D) {
+					for blkheaderid := uint64(0); d.PeekBits(8) == blkheaderid; blkheaderid++ {
+						d.FieldStruct("XlogRecordBlockHeader", func(d *decode.D) {
+							/* block reference ID */
+							d.FieldU8("id", d.AssertU(blkheaderid))
+							/* fork within the relation, and flags */
+							fork_flags := d.FieldU8("fork_flags")
+							/* number of payload bytes (not including page image) */
+							lenghts = append(lenghts, d.FieldU16("data_length"))
+							if fork_flags&BKPBLOCK_HAS_IMAGE != 0 {
+								d.FieldStruct("XLogRecordBlockImageHeader", func(d *decode.D) {
+									/* number of page image bytes */
+									d.FieldU16("length")
+									/* number of bytes before "hole" */
+									d.FieldU16("hole_offset")
+									/* flag bits, see below */
+									bimg_info := d.FieldU8("bimg_info")
+									d.FieldRawLen("padding", int64(d.AlignBits(16)))
+									if bimg_info&BKPIMAGE_HAS_HOLE != 0 &&
+										bimg_info&BKPIMAGE_IS_COMPRESSED != 0 {
+										d.FieldU16("hole_length")
+									}
+								})
+							}
+							if fork_flags&BKPBLOCK_SAME_REL == 0 {
+								d.FieldStruct("RelFileNode", func(d *decode.D) {
+									/* tablespace */
+									d.FieldU32("spcNode")
+									/* database */
+									d.FieldU32("dbNode")
+									/* relation */
+									d.FieldU32("relNode")
+								})
+								d.FieldU32("BlockNum")
+							}
+						})
+					}
+				})
+				if d.PeekBits(8) == 0xff {
+					d.FieldStruct("XLogRecordDataHeaderShort", func(d *decode.D) {
+						d.FieldU8("id", d.AssertU(0xff))
+						lenghts = append(lenghts, d.FieldU8("data_length"))
+					})
+				}
+
+				d.FieldArray("data", func(d *decode.D) {
+					for _, x := range lenghts {
+						pos := uint64(d.Pos()) >> 3
+						if pos < header_pos && (header_pos < pos+x) {
+							d.FieldRawLen("data", int64((header_pos-pos)*8))
+							header := pageHeaders.FieldStruct("XLogPageHeaderData", decodeXLogPageHeaderData)
+							header.FieldGet("xlp_rem_len").TryScalarFn(d.ValidateU(record_end - header_pos))
+							d.FieldRawLen("data", int64((x+pos-header_pos)*8))
+						} else {
+							d.FieldRawLen("data", int64(x*8))
+						}
+					}
+				})
+
+				d.FieldRawLen("ending_padding", int64(d.AlignBits(64)))
+			})
+		}
+	})
+
+	return nil
+}

format/postgres/pgwal_page.go

@@ -0,0 +1,28 @@
+package postgres
+
+import (
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/format/registry"
+	"github.com/wader/fq/pkg/decode"
+	_ "github.com/wader/fq/pkg/scalar"
+)
+
+func init() {
+	registry.MustRegister(decode.Format{
+		Name:        format.PGWALPAGE,
+		Description: "PostgreSQL write-ahead page",
+		DecodeFn:    walpageDecode,
+	})
+}
+
+//const XLOG_BLCKSZ = 8192
+
+func walpageDecode(d *decode.D, in interface{}) interface{} {
+
+	d.Endian = decode.LittleEndian
+
+	pageHeaders := d.FieldArrayValue("XLogPageHeaders")
+	_ = pageHeaders.FieldStruct("XLogPageHeaderData", decodeXLogPageHeaderData)
+
+	return nil
+}

go.mod

@@ -29,6 +29,28 @@ require (
 )
 
 require (
+	github.com/cilium/ebpf v0.7.0 // indirect
+	github.com/cosiner/argv v0.1.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/derekparker/trie v0.0.0-20200317170641-1fdf38b7b0e9 // indirect
+	github.com/go-delve/delve v1.8.0 // indirect
+	github.com/google/go-dap v0.6.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/itchyny/timefmt-go v0.1.3 // indirect
-	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/peterh/liner v1.2.1 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.starlark.net v0.0.0-20211203141949-70c0e40ae128 // indirect
+	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )