-
Notifications
You must be signed in to change notification settings - Fork 1.2k
/
block_header.go
183 lines (156 loc) · 5.49 KB
/
block_header.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
package mergeset
import (
"fmt"
"sort"
"unsafe"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
)
type blockHeader struct {
// common prefix for all the items in the block.
commonPrefix []byte
// The first item.
firstItem []byte
// Whether commonPrefix and firstItem point to external data.
noCopy bool
// Marshal type used for block compression.
marshalType marshalType
// The number of items in the block, excluding the first item.
itemsCount uint32
// The offset of the items block.
itemsBlockOffset uint64
// The offset of the lens block.
lensBlockOffset uint64
// The size of the items block.
itemsBlockSize uint32
// The size of the lens block.
lensBlockSize uint32
}
func (bh *blockHeader) SizeBytes() int {
return int(unsafe.Sizeof(*bh)) + cap(bh.commonPrefix) + cap(bh.firstItem)
}
func (bh *blockHeader) Reset() {
if bh.noCopy {
bh.commonPrefix = nil
bh.firstItem = nil
} else {
bh.commonPrefix = bh.commonPrefix[:0]
bh.firstItem = bh.firstItem[:0]
}
bh.marshalType = marshalTypePlain
bh.itemsCount = 0
bh.itemsBlockOffset = 0
bh.lensBlockOffset = 0
bh.itemsBlockSize = 0
bh.lensBlockSize = 0
}
func (bh *blockHeader) Marshal(dst []byte) []byte {
dst = encoding.MarshalBytes(dst, bh.commonPrefix)
dst = encoding.MarshalBytes(dst, bh.firstItem)
dst = append(dst, byte(bh.marshalType))
dst = encoding.MarshalUint32(dst, bh.itemsCount)
dst = encoding.MarshalUint64(dst, bh.itemsBlockOffset)
dst = encoding.MarshalUint64(dst, bh.lensBlockOffset)
dst = encoding.MarshalUint32(dst, bh.itemsBlockSize)
dst = encoding.MarshalUint32(dst, bh.lensBlockSize)
return dst
}
// UnmarshalNoCopy unmarshals bh from src without copying the data from src.
//
// The src must remain unchanged while bh is in use.
func (bh *blockHeader) UnmarshalNoCopy(src []byte) ([]byte, error) {
bh.noCopy = true
// Unmarshal commonPrefix
cp, nSize := encoding.UnmarshalBytes(src)
if nSize <= 0 {
return src, fmt.Errorf("cannot unmarshal commonPrefix")
}
src = src[nSize:]
bh.commonPrefix = cp[:len(cp):len(cp)]
// Unmarshal firstItem
fi, nSize := encoding.UnmarshalBytes(src)
if nSize <= 0 {
return src, fmt.Errorf("cannot unmarshal firstItem")
}
src = src[nSize:]
bh.firstItem = fi[:len(fi):len(fi)]
// Unmarshal marshalType
if len(src) == 0 {
return src, fmt.Errorf("cannot unmarshal marshalType from zero bytes")
}
bh.marshalType = marshalType(src[0])
src = src[1:]
if err := checkMarshalType(bh.marshalType); err != nil {
return src, fmt.Errorf("unexpected marshalType: %w", err)
}
// Unmarshal itemsCount
if len(src) < 4 {
return src, fmt.Errorf("cannot unmarshal itemsCount from %d bytes; need at least %d bytes", len(src), 4)
}
bh.itemsCount = encoding.UnmarshalUint32(src)
src = src[4:]
// Unmarshal itemsBlockOffset
if len(src) < 8 {
return src, fmt.Errorf("cannot unmarshal itemsBlockOffset from %d bytes; neet at least %d bytes", len(src), 8)
}
bh.itemsBlockOffset = encoding.UnmarshalUint64(src)
src = src[8:]
// Unmarshal lensBlockOffset
if len(src) < 8 {
return src, fmt.Errorf("cannot unmarshal lensBlockOffset from %d bytes; need at least %d bytes", len(src), 8)
}
bh.lensBlockOffset = encoding.UnmarshalUint64(src)
src = src[8:]
// Unmarshal itemsBlockSize
if len(src) < 4 {
return src, fmt.Errorf("cannot unmarshal itemsBlockSize from %d bytes; need at least %d bytes", len(src), 4)
}
bh.itemsBlockSize = encoding.UnmarshalUint32(src)
src = src[4:]
// Unmarshal lensBlockSize
if len(src) < 4 {
return src, fmt.Errorf("cannot unmarshal lensBlockSize from %d bytes; need at least %d bytes", len(src), 4)
}
bh.lensBlockSize = encoding.UnmarshalUint32(src)
src = src[4:]
if bh.itemsCount <= 0 {
return src, fmt.Errorf("itemsCount must be bigger than 0; got %d", bh.itemsCount)
}
if bh.itemsBlockSize > 2*maxInmemoryBlockSize {
return src, fmt.Errorf("too big itemsBlockSize; got %d; cannot exceed %d", bh.itemsBlockSize, 2*maxInmemoryBlockSize)
}
if bh.lensBlockSize > 2*8*maxInmemoryBlockSize {
return src, fmt.Errorf("too big lensBlockSize; got %d; cannot exceed %d", bh.lensBlockSize, 2*8*maxInmemoryBlockSize)
}
return src, nil
}
// unmarshalBlockHeadersNoCopy unmarshals all the block headers from src,
// appends them to dst and returns the appended result.
//
// Block headers must be sorted by bh.firstItem.
//
// It is expected that src remains unchanged while rhe returned blocks are in use.
func unmarshalBlockHeadersNoCopy(dst []blockHeader, src []byte, blockHeadersCount int) ([]blockHeader, error) {
if blockHeadersCount <= 0 {
logger.Panicf("BUG: blockHeadersCount must be greater than 0; got %d", blockHeadersCount)
}
dstLen := len(dst)
dst = slicesutil.SetLength(dst, dstLen+blockHeadersCount)
for i := 0; i < blockHeadersCount; i++ {
tail, err := dst[dstLen+i].UnmarshalNoCopy(src)
if err != nil {
return dst, fmt.Errorf("cannot unmarshal block header #%d out of %d: %w", i, blockHeadersCount, err)
}
src = tail
}
if len(src) > 0 {
return dst, fmt.Errorf("unexpected non-zero tail left after unmarshaling %d block headers; len(tail)=%d", blockHeadersCount, len(src))
}
newBHS := dst[dstLen:]
// Verify that block headers are sorted by firstItem.
if !sort.SliceIsSorted(newBHS, func(i, j int) bool { return string(newBHS[i].firstItem) < string(newBHS[j].firstItem) }) {
return dst, fmt.Errorf("block headers must be sorted by firstItem; unmarshaled unsorted block headers: %#v", newBHS)
}
return dst, nil
}