Skip to content

Commit 7fde12a

Browse files
jbowensjbowens
committed
colblk: mark blob handles as external values
In data blocks, set the 'isValueExternal' bit for rows encoding handles to values stored in external blob files. Previously only value block handles set this bit.
1 parent ebfaa8f commit 7fde12a

File tree

4 files changed

+187
-34
lines changed

4 files changed

+187
-34
lines changed

‎sstable/colblk/data_block.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -581,7 +581,7 @@ func (w *DataBlockEncoder) Add(
581581
w.isObsolete.Set(w.rows)
582582
}
583583
w.trailers.Set(w.rows, uint64(ikey.Trailer))
584-
if valuePrefix.IsValueBlockHandle() {
584+
if !valuePrefix.IsInPlaceValue() {
585585
w.isValueExternal.Set(w.rows)
586586
// Write the value with the value prefix byte preceding the value.
587587
w.valuePrefixTmp[0] = byte(valuePrefix)

‎sstable/colblk/data_block_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ func TestDataBlock(t *testing.T) {
7070
vp := block.InPlaceValuePrefix(kcmp.PrefixEqual())
7171
if strings.HasPrefix(valueString, "valueHandle") {
7272
vp = block.ValueBlockHandlePrefix(kcmp.PrefixEqual(), 0)
73+
} else if strings.HasPrefix(valueString, "blobHandle") {
74+
vp = block.BlobValueHandlePrefix(kcmp.PrefixEqual(), 0)
7375
}
7476
if kcmp.UserKeyComparison == 0 && prevKey.Kind() != base.InternalKeyKindMerge {
7577
isObsolete = true

‎sstable/colblk/testdata/data_block/external_value‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,3 +337,151 @@ next-prefix
337337
----
338338
seek-ge blockprefix_banana@73: blockprefix_banana@72:mock external value
339339
next-prefix: blockprefix_coconut:coconut
340+
341+
init
342+
----
343+
size=51:
344+
0: prefixes: prefixbytes(16): 0 keys
345+
1: suffixes: bytes: 0 rows set; 0 bytes in data
346+
2: trailers: uint: 0 rows
347+
3: prefix changed: bitmap
348+
4: values: bytes: 0 rows set; 0 bytes in data
349+
5: is-value-ext: bitmap
350+
6: is-obsolete: bitmap
351+
352+
write
353+
apple@98#0,SET:apple98
354+
apple@52#0,SET:blobHandle-apple52
355+
apple@23#0,SET:blobHandle-apple23
356+
apple@11#0,SETWITHDEL:blobHandle-apple11
357+
banana@94#245,SETWITHDEL:banana94
358+
banana@93#244,DEL:
359+
----
360+
size=234:
361+
0: prefixes: prefixbytes(16): 6 keys
362+
1: suffixes: bytes: 6 rows set; 18 bytes in data
363+
2: trailers: uint: 6 rows
364+
3: prefix changed: bitmap
365+
4: values: bytes: 6 rows set; 72 bytes in data
366+
5: is-value-ext: bitmap
367+
6: is-obsolete: bitmap
368+
369+
finish
370+
----
371+
LastKey: banana@93#244,DEL
372+
data block header
373+
├── columnar block header
374+
│ ├── 000-004: x 09000000 # maximum key length: 9
375+
│ ├── 004-005: x 01 # version 1
376+
│ ├── 005-007: x 0700 # 7 columns
377+
│ ├── 007-011: x 06000000 # 6 rows
378+
│ ├── 011-012: b 00000100 # col 0: prefixbytes
379+
│ ├── 012-016: x 2e000000 # col 0: page start 46
380+
│ ├── 016-017: b 00000011 # col 1: bytes
381+
│ ├── 017-021: x 43000000 # col 1: page start 67
382+
│ ├── 021-022: b 00000010 # col 2: uint
383+
│ ├── 022-026: x 5d000000 # col 2: page start 93
384+
│ ├── 026-027: b 00000001 # col 3: bool
385+
│ ├── 027-031: x 6a000000 # col 3: page start 106
386+
│ ├── 031-032: b 00000011 # col 4: bytes
387+
│ ├── 032-036: x 80000000 # col 4: page start 128
388+
│ ├── 036-037: b 00000001 # col 5: bool
389+
│ ├── 037-041: x d0000000 # col 5: page start 208
390+
│ ├── 041-042: b 00000001 # col 6: bool
391+
│ └── 042-046: x e8000000 # col 6: page start 232
392+
├── data for column 0 (prefixbytes)
393+
│ ├── 046-047: x 04 # bundle size: 16
394+
│ ├── offsets table
395+
│ │ ├── 047-048: x 01 # encoding: 1b
396+
│ │ ├── 048-049: x 00 # data[0] = 0 [56 overall]
397+
│ │ ├── 049-050: x 00 # data[1] = 0 [56 overall]
398+
│ │ ├── 050-051: x 05 # data[2] = 5 [61 overall]
399+
│ │ ├── 051-052: x 05 # data[3] = 5 [61 overall]
400+
│ │ ├── 052-053: x 05 # data[4] = 5 [61 overall]
401+
│ │ ├── 053-054: x 05 # data[5] = 5 [61 overall]
402+
│ │ ├── 054-055: x 0b # data[6] = 11 [67 overall]
403+
│ │ └── 055-056: x 0b # data[7] = 11 [67 overall]
404+
│ └── data
405+
│ ├── 056-056: x # data[00]: (block prefix)
406+
│ ├── 056-056: x # data[01]: (bundle prefix)
407+
│ ├── 056-061: x 6170706c65 # data[02]: apple
408+
│ ├── 061-061: x # data[03]: .....
409+
│ ├── 061-061: x # data[04]: .....
410+
│ ├── 061-061: x # data[05]: .....
411+
│ ├── 061-067: x 62616e616e61 # data[06]: banana
412+
│ └── 067-067: x # data[07]: ......
413+
├── data for column 1 (bytes)
414+
│ ├── offsets table
415+
│ │ ├── 067-068: x 01 # encoding: 1b
416+
│ │ ├── 068-069: x 00 # data[0] = 0 [75 overall]
417+
│ │ ├── 069-070: x 03 # data[1] = 3 [78 overall]
418+
│ │ ├── 070-071: x 06 # data[2] = 6 [81 overall]
419+
│ │ ├── 071-072: x 09 # data[3] = 9 [84 overall]
420+
│ │ ├── 072-073: x 0c # data[4] = 12 [87 overall]
421+
│ │ ├── 073-074: x 0f # data[5] = 15 [90 overall]
422+
│ │ └── 074-075: x 12 # data[6] = 18 [93 overall]
423+
│ └── data
424+
│ ├── 075-078: x 403938 # data[0]: @98
425+
│ ├── 078-081: x 403532 # data[1]: @52
426+
│ ├── 081-084: x 403233 # data[2]: @23
427+
│ ├── 084-087: x 403131 # data[3]: @11
428+
│ ├── 087-090: x 403934 # data[4]: @94
429+
│ └── 090-093: x 403933 # data[5]: @93
430+
├── data for column 2 (uint)
431+
│ ├── 093-094: x 02 # encoding: 2b
432+
│ ├── 094-096: x 0100 # data[0] = 1
433+
│ ├── 096-098: x 0100 # data[1] = 1
434+
│ ├── 098-100: x 0100 # data[2] = 1
435+
│ ├── 100-102: x 1200 # data[3] = 18
436+
│ ├── 102-104: x 12f5 # data[4] = 62738
437+
│ └── 104-106: x 00f4 # data[5] = 62464
438+
├── data for column 3 (bool)
439+
│ ├── 106-107: x 00 # default bitmap encoding
440+
│ ├── 107-112: x 0000000000 # padding to align to 64-bit boundary
441+
│ ├── 112-120: b 0001000100000000000000000000000000000000000000000000000000000000 # bitmap word 0
442+
│ └── 120-128: b 0000000100000000000000000000000000000000000000000000000000000000 # bitmap summary word 0-63
443+
├── data for column 4 (bytes)
444+
│ ├── offsets table
445+
│ │ ├── 128-129: x 01 # encoding: 1b
446+
│ │ ├── 129-130: x 00 # data[0] = 0 [136 overall]
447+
│ │ ├── 130-131: x 07 # data[1] = 7 [143 overall]
448+
│ │ ├── 131-132: x 1a # data[2] = 26 [162 overall]
449+
│ │ ├── 132-133: x 2d # data[3] = 45 [181 overall]
450+
│ │ ├── 133-134: x 40 # data[4] = 64 [200 overall]
451+
│ │ ├── 134-135: x 48 # data[5] = 72 [208 overall]
452+
│ │ └── 135-136: x 48 # data[6] = 72 [208 overall]
453+
│ └── data
454+
│ ├── 136-143: x 6170706c653938 # data[0]: apple98
455+
│ ├── 143-153: x 60626c6f6248616e646c # data[1]: `blobHandle-apple52
456+
│ ├── 153-162: x 652d6170706c653532 # (continued...)
457+
│ ├── 162-172: x 60626c6f6248616e646c # data[2]: `blobHandle-apple23
458+
│ ├── 172-181: x 652d6170706c653233 # (continued...)
459+
│ ├── 181-191: x 60626c6f6248616e646c # data[3]: `blobHandle-apple11
460+
│ ├── 191-200: x 652d6170706c653131 # (continued...)
461+
│ ├── 200-208: x 62616e616e613934 # data[4]: banana94
462+
│ └── 208-208: x # data[5]:
463+
├── data for column 5 (bool)
464+
│ ├── 208-209: x 00 # default bitmap encoding
465+
│ ├── 209-216: x 00000000000000 # padding to align to 64-bit boundary
466+
│ ├── 216-224: b 0000111000000000000000000000000000000000000000000000000000000000 # bitmap word 0
467+
│ └── 224-232: b 0000000100000000000000000000000000000000000000000000000000000000 # bitmap summary word 0-63
468+
├── data for column 6 (bool)
469+
│ └── 232-233: x 01 # zero bitmap encoding
470+
└── 233-234: x 00 # block padding byte
471+
472+
iter
473+
first
474+
next
475+
next
476+
next
477+
next
478+
next
479+
next
480+
----
481+
first: apple@98:apple98
482+
next: apple@52:mock external value
483+
next: apple@23:mock external value
484+
next: apple@11:mock external value
485+
next: banana@94:banana94
486+
next: banana@93:
487+
next: .

‎sstable/testdata/writer_blob_value_handles‎

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ blob-separated-values: num-values 4
1010
layout
1111
----
1212
sstable
13-
├── data offset: 0 length: 133
13+
├── data offset: 0 length: 154
1414
│ ├── data block header
1515
│ │ ├── columnar block header
1616
│ │ │ ├── 000-004: x 03000000 # maximum key length: 3
@@ -28,9 +28,9 @@ sstable
2828
│ │ │ ├── 031-032: b 00000011 # col 4: bytes
2929
│ │ │ ├── 032-036: x 68000000 # col 4: page start 104
3030
│ │ │ ├── 036-037: b 00000001 # col 5: bool
31-
│ │ │ ├── 037-041: x 82000000 # col 5: page start 130
31+
│ │ │ ├── 037-041: x 86000000 # col 5: page start 134
3232
│ │ │ ├── 041-042: b 00000001 # col 6: bool
33-
│ │ │ └── 042-046: x 83000000 # col 6: page start 131
33+
│ │ │ └── 042-046: x 98000000 # col 6: page start 152
3434
│ │ ├── data for column 0 (prefixbytes)
3535
│ │ │ ├── 046-047: x 04 # bundle size: 16
3636
│ │ │ ├── offsets table
@@ -82,33 +82,36 @@ sstable
8282
│ │ │ ├── offsets table
8383
│ │ │ │ ├── 104-105: x 01 # encoding: 1b
8484
│ │ │ │ ├── 105-106: x 00 # data[0] = 0 [111 overall]
85-
│ │ │ │ ├── 106-107: x 04 # data[1] = 4 [115 overall]
86-
│ │ │ │ ├── 107-108: x 09 # data[2] = 9 [120 overall]
87-
│ │ │ │ ├── 108-109: x 09 # data[3] = 9 [120 overall]
88-
│ │ │ │ ├── 109-110: x 0e # data[4] = 14 [125 overall]
89-
│ │ │ │ └── 110-111: x 13 # data[5] = 19 [130 overall]
85+
│ │ │ │ ├── 106-107: x 05 # data[1] = 5 [116 overall]
86+
│ │ │ │ ├── 107-108: x 0b # data[2] = 11 [122 overall]
87+
│ │ │ │ ├── 108-109: x 0b # data[3] = 11 [122 overall]
88+
│ │ │ │ ├── 109-110: x 11 # data[4] = 17 [128 overall]
89+
│ │ │ │ └── 110-111: x 17 # data[5] = 23 [134 overall]
9090
│ │ │ └── data
91-
│ │ │ ├── 111-115: x 0064010a # data[0]: "\x00d\x01\n"
92-
│ │ │ ├── 115-120: x 00c801026e # data[1]: "\x00\xc8\x01\x02n"
93-
│ │ │ ├── 120-120: x # data[2]:
94-
│ │ │ ├── 120-125: x 01c8010000 # data[3]: "\x01\xc8\x01\x00\x00"
95-
│ │ │ └── 125-130: x 016701a602 # data[4]: "\x01g\x01\xa6\x02"
91+
│ │ │ ├── 111-116: x 470064010a # data[0]: "G\x00d\x01\n"
92+
│ │ │ ├── 116-122: x 4700c801026e # data[1]: "G\x00\xc8\x01\x02n"
93+
│ │ │ ├── 122-122: x # data[2]:
94+
│ │ │ ├── 122-128: x 6701c8010000 # data[3]: "g\x01\xc8\x01\x00\x00"
95+
│ │ │ └── 128-134: x 67016701a602 # data[4]: "g\x01g\x01\xa6\x02"
9696
│ │ ├── data for column 5 (bool)
97-
│ │ │ └── 130-131: x 01 # zero bitmap encoding
97+
│ │ │ ├── 134-135: x 00 # default bitmap encoding
98+
│ │ │ ├── 135-136: x 00 # padding to align to 64-bit boundary
99+
│ │ │ ├── 136-144: b 0001101100000000000000000000000000000000000000000000000000000000 # bitmap word 0
100+
│ │ │ └── 144-152: b 0000000100000000000000000000000000000000000000000000000000000000 # bitmap summary word 0-63
98101
│ │ ├── data for column 6 (bool)
99-
│ │ │ └── 131-132: x 01 # zero bitmap encoding
100-
│ │ └── 132-133: x 00 # block padding byte
101-
│ ├── a@2#1,SET:hex:0064010a
102-
│ ├── b@5#7,SET:hex:00c801026e
102+
│ │ │ └── 152-153: x 01 # zero bitmap encoding
103+
│ │ └── 153-154: x 00 # block padding byte
104+
│ ├── a@2#1,SET:value handle {ValueLen:0 BlockNum:100 OffsetInBlock:1}
105+
│ ├── b@5#7,SET:value handle {ValueLen:0 BlockNum:200 OffsetInBlock:2}
103106
│ ├── b@4#3,DEL:
104-
│ ├── b@3#2,SET:hex:01c8010000
105-
│ ├── b@2#1,SET:hex:016701a602
106-
│ └── trailer [compression=none checksum=0x8f109d09]
107-
├── index offset: 138 length: 36
108-
│ ├── 00000 block:0/133
107+
│ ├── b@3#2,SET:value handle {ValueLen:1 BlockNum:200 OffsetInBlock:0}
108+
│ ├── b@2#1,SET:value handle {ValueLen:1 BlockNum:103 OffsetInBlock:1}
109+
│ └── trailer [compression=none checksum=0x196b9b28]
110+
├── index offset: 159 length: 36
111+
│ ├── 00000 block:0/154
109112
│ │
110-
│ └── trailer [compression=none checksum=0x863e7ff6]
111-
├── properties offset: 179 length: 570
113+
│ └── trailer [compression=none checksum=0xda1c8436]
114+
├── properties offset: 200 length: 570
112115
│ ├── 00000 obsolete-key (16) [restart]
113116
│ ├── 00016 pebble.colblk.schema (68)
114117
│ ├── 00084 pebble.num.values.in.blob-files (28)
@@ -131,17 +134,17 @@ sstable
131134
│ ├── 00547 rocksdb.raw.value.size (15)
132135
│ ├── restart points
133136
│ │ └── 00562 [restart 0]
134-
│ └── trailer [compression=none checksum=0x5aee2759]
135-
├── meta-index offset: 754 length: 33
136-
│ ├── 0000 rocksdb.properties block:179/570 [restart]
137+
│ └── trailer [compression=none checksum=0xaaf4ac72]
138+
├── meta-index offset: 775 length: 33
139+
│ ├── 0000 rocksdb.properties block:200/570 [restart]
137140
│ ├── restart points
138141
│ │ └── 00025 [restart 0]
139-
│ └── trailer [compression=none checksum=0x336f39a9]
140-
└── footer offset: 792 length: 57
142+
│ └── trailer [compression=none checksum=0x7216e2e4]
143+
└── footer offset: 813 length: 57
141144
├── 000 checksum type: crc32c
142-
├── 001 meta: offset=754, length=33
143-
├── 004 index: offset=138, length=36
144-
├── 041 footer checksum: 0x7c352f7a
145+
├── 001 meta: offset=775, length=33
146+
├── 004 index: offset=159, length=36
147+
├── 041 footer checksum: 0x1d06bfc3
145148
├── 045 version: 6
146149
└── 049 magic number: 0xf09faab3f09faab3
147150

0 commit comments

Comments
 (0)