Skip to content

Commit 209a45f

Browse files
annrpomannrpom
committed
tool: add support for inspecting sstable's blob value liveness index
The new blob value liveness index block in an sstable indicates which values in external blob files a sstable references. This commit adds support for inspecting the block within `sstable layout`. Fixes: #4889
1 parent 243cff4 commit 209a45f

File tree

3 files changed

+136
-2
lines changed

3 files changed

+136
-2
lines changed

‎sstable/layout.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -376,10 +376,15 @@ func (l *Layout) Describe(
376376
var decoder colblk.ReferenceLivenessBlockDecoder
377377
decoder.Init(h.BlockData())
378378
offset := 0
379-
for i := 0; i < decoder.BlockDecoder().Rows(); i++ {
379+
for i := range decoder.BlockDecoder().Rows() {
380380
value := decoder.LivenessAtReference(i)
381+
encs := DecodeBlobRefLivenessEncoding(value)
381382
length := len(value)
382-
tpNode.Childf("%05d %s (%d)", offset, value, length)
383+
parent := tpNode.Childf("%05d (%d)", offset, length)
384+
for _, enc := range encs {
385+
parent.Childf("block: %d, values size: %d, bitmap size: %d byte(s), bitmap: %08b",
386+
enc.BlockID, enc.ValuesSize, enc.BitmapSize, enc.Bitmap)
387+
}
383388
offset += length
384389
}
385390
}

‎sstable/testdata/writer_blob_value_handles‎

468 Bytes
Binary file not shown.

‎tool/testdata/sstable_layout‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3860,3 +3860,132 @@ sstable
38603860
├── properties offset: 333 length: 601
38613861
├── meta-index offset: 939 length: 59
38623862
└── footer offset: 1003 length: 53
3863+
3864+
sstable layout
3865+
testdata/find-val-sep-db/000008.sst
3866+
-v
3867+
----
3868+
000008.sst
3869+
sstable
3870+
├── data offset: 0 length: 102
3871+
│ ├── data block header
3872+
│ │ ├── columnar block header
3873+
│ │ │ ├── 000-001: x 02 # key schema header
3874+
│ │ │ ├── 001-005: x 04000000 # maximum key length: 4
3875+
│ │ │ ├── 005-006: x 01 # version 1
3876+
│ │ │ ├── 006-008: x 0900 # 9 columns
3877+
│ │ │ ├── 008-012: x 02000000 # 2 rows
3878+
│ │ │ ├── 012-013: b 00000100 # col 0: prefixbytes
3879+
│ │ │ ├── 013-017: x 39000000 # col 0: page start 57
3880+
│ │ │ ├── 017-018: b 00000010 # col 1: uint
3881+
│ │ │ ├── 018-022: x 45000000 # col 1: page start 69
3882+
│ │ │ ├── 022-023: b 00000010 # col 2: uint
3883+
│ │ │ ├── 023-027: x 46000000 # col 2: page start 70
3884+
│ │ │ ├── 027-028: b 00000011 # col 3: bytes
3885+
│ │ │ ├── 028-032: x 47000000 # col 3: page start 71
3886+
│ │ │ ├── 032-033: b 00000010 # col 4: uint
3887+
│ │ │ ├── 033-037: x 48000000 # col 4: page start 72
3888+
│ │ │ ├── 037-038: b 00000001 # col 5: bool
3889+
│ │ │ ├── 038-042: x 4e000000 # col 5: page start 78
3890+
│ │ │ ├── 042-043: b 00000011 # col 6: bytes
3891+
│ │ │ ├── 043-047: x 60000000 # col 6: page start 96
3892+
│ │ │ ├── 047-048: b 00000001 # col 7: bool
3893+
│ │ │ ├── 048-052: x 6e000000 # col 7: page start 110
3894+
│ │ │ ├── 052-053: b 00000001 # col 8: bool
3895+
│ │ │ └── 053-057: x 80000000 # col 8: page start 128
3896+
│ │ ├── data for column 0 (prefixbytes)
3897+
│ │ │ ├── 057-058: x 04 # bundle size: 16
3898+
│ │ │ ├── offsets table
3899+
│ │ │ │ ├── 058-059: x 01 # encoding: 1b
3900+
│ │ │ │ ├── 059-060: x 00 # data[0] = 0 [63 overall]
3901+
│ │ │ │ ├── 060-061: x 00 # data[1] = 0 [63 overall]
3902+
│ │ │ │ ├── 061-062: x 03 # data[2] = 3 [66 overall]
3903+
│ │ │ │ └── 062-063: x 06 # data[3] = 6 [69 overall]
3904+
│ │ │ └── data
3905+
│ │ │ ├── 063-063: x # data[00]: (block prefix)
3906+
│ │ │ ├── 063-063: x # data[01]: (bundle prefix)
3907+
│ │ │ ├── 063-066: x 656565 # data[02]: eee
3908+
│ │ │ └── 066-069: x 666666 # data[03]: fff
3909+
│ │ ├── data for column 1 (uint)
3910+
│ │ │ └── 069-070: x 00 # encoding: zero
3911+
│ │ ├── data for column 2 (uint)
3912+
│ │ │ └── 070-071: x 00 # encoding: zero
3913+
│ │ ├── data for column 3 (bytes)
3914+
│ │ │ ├── offsets table
3915+
│ │ │ │ └── 071-072: x 00 # encoding: zero
3916+
│ │ │ └── data
3917+
│ │ │ ├── 072-072: x # data[0]:
3918+
│ │ │ └── 072-072: x # data[1]:
3919+
│ │ ├── data for column 4 (uint)
3920+
│ │ │ ├── 072-073: x 02 # encoding: 2b
3921+
│ │ │ ├── 073-074: x 00 # padding (aligning to 16-bit boundary)
3922+
│ │ │ ├── 074-076: x 010e # data[0] = 3585
3923+
│ │ │ └── 076-078: x 010f # data[1] = 3841
3924+
│ │ ├── data for column 5 (bool)
3925+
│ │ │ ├── 078-079: x 00 # default bitmap encoding
3926+
│ │ │ ├── 079-080: x 00 # padding to align to 64-bit boundary
3927+
│ │ │ ├── 080-088: b 0000001100000000000000000000000000000000000000000000000000000000 # bitmap word 0
3928+
│ │ │ └── 088-096: b 0000000100000000000000000000000000000000000000000000000000000000 # bitmap summary word 0-63
3929+
│ │ ├── data for column 6 (bytes)
3930+
│ │ │ ├── offsets table
3931+
│ │ │ │ ├── 096-097: x 01 # encoding: 1b
3932+
│ │ │ │ ├── 097-098: x 00 # data[0] = 0 [100 overall]
3933+
│ │ │ │ ├── 098-099: x 05 # data[1] = 5 [105 overall]
3934+
│ │ │ │ └── 099-100: x 0a # data[2] = 10 [110 overall]
3935+
│ │ │ └── data
3936+
│ │ │ ├── 100-105: x 4000060000 # data[0]: "@\x00\x06\x00\x00"
3937+
│ │ │ └── 105-110: x 4000070001 # data[1]: "@\x00\a\x00\x01"
3938+
│ │ ├── data for column 7 (bool)
3939+
│ │ │ ├── 110-111: x 00 # default bitmap encoding
3940+
│ │ │ ├── 111-112: x 00 # padding to align to 64-bit boundary
3941+
│ │ │ ├── 112-120: b 0000001100000000000000000000000000000000000000000000000000000000 # bitmap word 0
3942+
│ │ │ └── 120-128: b 0000000100000000000000000000000000000000000000000000000000000000 # bitmap summary word 0-63
3943+
│ │ ├── data for column 8 (bool)
3944+
│ │ │ └── 128-129: x 01 # zero bitmap encoding
3945+
│ │ └── 129-130: x 00 # block padding byte
3946+
│ ├── eee\x00#14,SET [626c6f622068616e646c65202866302c626c6b302c6964302c6c656e3629]
3947+
│ ├── fff\x00#15,SET [626c6f622068616e646c65202866302c626c6b302c6964312c6c656e3729]
3948+
│ └── trailer [compression=snappy checksum=0x56aca881]
3949+
├── index offset: 107 length: 36
3950+
│ ├── 00000 block:0/102
3951+
│ │
3952+
│ └── trailer [compression=none checksum=0x214f31b5]
3953+
├── blob-reference-index offset: 148 length: 20
3954+
│ ├── 00000 (4)
3955+
│ │ └── block: 0, values size: 13, bitmap size: 1 byte(s), bitmap: [00000011]
3956+
│ └── trailer [compression=none checksum=0x63e76459]
3957+
├── properties offset: 173 length: 508
3958+
│ ├── 00000 obsolete-key (13)
3959+
│ ├── 00013 pebble.colblk.schema (25)
3960+
│ ├── 00038 pebble.num.values.in.blob-files (32)
3961+
│ ├── 00070 rocksdb.block.based.table.index.type (40)
3962+
│ ├── 00110 rocksdb.comparator (44)
3963+
│ ├── 00154 rocksdb.compression (25)
3964+
│ ├── 00179 rocksdb.compression_options (122)
3965+
│ ├── 00301 rocksdb.data.size (18)
3966+
│ ├── 00319 rocksdb.deleted.keys (21)
3967+
│ ├── 00340 rocksdb.filter.size (20)
3968+
│ ├── 00360 rocksdb.index.size (19)
3969+
│ ├── 00379 rocksdb.merge.operands (23)
3970+
│ ├── 00402 rocksdb.merge.operator (40)
3971+
│ ├── 00442 rocksdb.num.data.blocks (24)
3972+
│ ├── 00466 rocksdb.num.entries (20)
3973+
│ ├── 00486 rocksdb.num.range-deletions (28)
3974+
│ ├── 00514 rocksdb.property.collectors (41)
3975+
│ ├── 00555 rocksdb.raw.key.size (21)
3976+
│ ├── 00576 rocksdb.raw.value.size (23)
3977+
│ └── trailer [compression=snappy checksum=0x5bfa586b]
3978+
├── meta-index offset: 686 length: 72
3979+
│ ├── 0000 pebble.blob_ref_index block:148/20
3980+
│ │
3981+
│ ├── 0001 rocksdb.properties block:173/508
3982+
│ │
3983+
│ └── trailer [compression=none checksum=0x4fcb0b70]
3984+
└── footer offset: 763 length: 61
3985+
├── 000 checksum type: crc32c
3986+
├── 001 meta: offset=686, length=72
3987+
├── 004 index: offset=107, length=36
3988+
├── 041 attributes: [BlobValues,PointKeys]
3989+
├── 045 footer checksum: 0x43f32cc0
3990+
├── 049 version: 7
3991+
└── 053 magic number: 0xf09faab3f09faab3

0 commit comments

Comments
 (0)