-
Notifications
You must be signed in to change notification settings - Fork 2
/
seekable.go
231 lines (181 loc) · 7.81 KB
/
seekable.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
package seekable
import (
"encoding/binary"
"fmt"
"math"
"go.uber.org/zap/zapcore"
)
const (
/*
The format consists of a number of frames (Zstandard compressed frames and skippable frames), followed by a final skippable frame at the end containing the seek table.
Seek Table Format
The structure of the seek table frame is as follows:
|`Skippable_Magic_Number`|`Frame_Size`|`[Seek_Table_Entries]`|`Seek_Table_Footer`|
|------------------------|------------|----------------------|-------------------|
| 4 bytes | 4 bytes | 8-12 bytes each | 9 bytes |
Skippable_Magic_Number
Value: 0x184D2A5E.
This is for compatibility with Zstandard skippable frames: https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#skippable-frames.
Since it is legal for other Zstandard skippable frames to use the same
magic number, it is not recommended for a decoder to recognize frames
solely on this.
Frame_Size
The total size of the skippable frame, not including the `Skippable_Magic_Number` or `Frame_Size`.
This is for compatibility with Zstandard skippable frames: https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#skippable-frames.
https://github.com/facebook/zstd/blob/dev/contrib/seekable_format/zstd_seekable_compression_format.md
*/
SkippableFrameMagic = 0x184D2A50
seekableMagicNumber = 0x8F92EAB1
seekTableFooterOffset = 9
frameSizeFieldSize = 4
skippableMagicNumberFieldSize = 4
seekableTag = 0xE
)
/*
SeekTableDescriptor is a Go representation of a bitfiled.
A bitfield describing the format of the seek table.
| Bit number | Field name |
| ---------- | ---------- |
| 7 | `Checksum_Flag` |
| 6-2 | `Reserved_Bits` |
| 1-0 | `Unused_Bits` |
While only `Checksum_Flag` currently exists, there are 7 other bits in this field that can be used for future changes to the format,
for example the addition of inline dictionaries.
*/
type SeekTableDescriptor struct {
// If the checksum flag is set, each of the seek table entries contains a 4 byte checksum
// of the uncompressed data contained in its frame.
ChecksumFlag bool
}
func (d *SeekTableDescriptor) MarshalLogObject(enc zapcore.ObjectEncoder) error {
enc.AddBool("ChecksumFlag", d.ChecksumFlag)
return nil
}
/*
SeekTableFooter is the footer of a seekable ZSTD stream.
The seek table footer format is as follows:
|`Number_Of_Frames`|`Seek_Table_Descriptor`|`Seekable_Magic_Number`|
|------------------|-----------------------|-----------------------|
| 4 bytes | 1 byte | 4 bytes |
https://github.com/facebook/zstd/blob/dev/contrib/seekable_format/zstd_seekable_compression_format.md#seek_table_footer
*/
type SeekTableFooter struct {
// The number of stored frames in the data.
NumberOfFrames uint32
// A bitfield describing the format of the seek table.
SeekTableDescriptor SeekTableDescriptor
// Value : 0x8F92EAB1.
SeekableMagicNumber uint32
}
func (f *SeekTableFooter) marshalBinaryInline(dst []byte) {
binary.LittleEndian.PutUint32(dst[0:], f.NumberOfFrames)
if f.SeekTableDescriptor.ChecksumFlag {
dst[4] |= 1 << 7
}
binary.LittleEndian.PutUint32(dst[5:], seekableMagicNumber)
}
func (f *SeekTableFooter) MarshalBinary() ([]byte, error) {
dst := make([]byte, seekTableFooterOffset)
f.marshalBinaryInline(dst)
return dst, nil
}
func (f *SeekTableFooter) MarshalLogObject(enc zapcore.ObjectEncoder) error {
enc.AddUint32("NumberOfFrames", f.NumberOfFrames)
if err := enc.AddObject("SeekTableDescriptor", &f.SeekTableDescriptor); err != nil {
return err
}
enc.AddUint32("SeekableMagicNumber", f.SeekableMagicNumber)
return nil
}
func (f *SeekTableFooter) UnmarshalBinary(p []byte) error {
if len(p) != seekTableFooterOffset {
return fmt.Errorf("footer length mismatch %d vs %d", len(p), seekTableFooterOffset)
}
f.NumberOfFrames = binary.LittleEndian.Uint32(p[0:])
f.SeekTableDescriptor.ChecksumFlag = (p[4] & (1 << 7)) > 0
f.SeekableMagicNumber = binary.LittleEndian.Uint32(p[5:])
if f.SeekableMagicNumber != seekableMagicNumber {
return fmt.Errorf("footer magic mismatch %d vs %d", f.SeekableMagicNumber, seekableMagicNumber)
}
return nil
}
/*
SeekTableEntry is an element of the Seek Table describing each of the ZSTD-compressed frames in the stream.
`Seek_Table_Entries` consists of `Number_Of_Frames` (one for each frame in the data, not including the seek table frame) entries of the following form, in sequence:
|`Compressed_Size`|`Decompressed_Size`|`[Checksum]`|
|-----------------|-------------------|------------|
| 4 bytes | 4 bytes | 4 bytes |
https://github.com/facebook/zstd/blob/dev/contrib/seekable_format/zstd_seekable_compression_format.md#seek_table_entries
*/
type SeekTableEntry struct {
// The compressed size of the frame.
// The cumulative sum of the `Compressed_Size` fields of frames `0` to `i` gives the offset in the compressed file of frame `i+1`.
CompressedSize uint32
// The size of the decompressed data contained in the frame. For skippable or otherwise empty frames, this value is 0.
DecompressedSize uint32
// Only present if `Checksum_Flag` is set in the `Seek_Table_Descriptor`. Value : the least significant 32 bits of the XXH64 digest of the uncompressed data, stored in little-endian format.
Checksum uint32
}
func (e *SeekTableEntry) marshalBinaryInline(dst []byte) {
binary.LittleEndian.PutUint32(dst[0:], e.CompressedSize)
binary.LittleEndian.PutUint32(dst[4:], e.DecompressedSize)
binary.LittleEndian.PutUint32(dst[8:], e.Checksum)
}
func (e *SeekTableEntry) MarshalBinary() ([]byte, error) {
dst := make([]byte, 12)
e.marshalBinaryInline(dst)
return dst, nil
}
func (e *SeekTableEntry) MarshalLogObject(enc zapcore.ObjectEncoder) error {
enc.AddUint32("CompressedSize", e.CompressedSize)
enc.AddUint32("DecompressedSize", e.DecompressedSize)
enc.AddUint32("Checksum", e.Checksum)
return nil
}
func (e *SeekTableEntry) UnmarshalBinary(p []byte) error {
if len(p) < 8 {
return fmt.Errorf("entry length mismatch %d vs %d", len(p), 8)
}
e.CompressedSize = binary.LittleEndian.Uint32(p[0:])
e.DecompressedSize = binary.LittleEndian.Uint32(p[4:])
if len(p) >= 12 {
e.Checksum = binary.LittleEndian.Uint32(p[8:])
}
return nil
}
/*
CreateSkippableFrame returns a payload formatted as a ZSDT skippable frame.
| `Magic_Number` | `Frame_Size` | `User_Data` |
|:--------------:|:------------:|:-----------:|
| 4 bytes | 4 bytes | n bytes |
Skippable frames allow the insertion of user-defined metadata
into a flow of concatenated frames.
Magic_Number
4 Bytes, __little-endian__ format.
Value : 0x184D2A5?, which means any value from 0x184D2A50 to 0x184D2A5F.
All 16 values are valid to identify a skippable frame.
This specification doesn't detail any specific tagging for skippable frames.
Frame_Size
This is the size, in bytes, of the following `User_Data`
(without including the magic number nor the size field itself).
This field is represented using 4 Bytes, __little-endian__ format, unsigned 32-bits.
This means `User_Data` can’t be bigger than (2^32-1) bytes.
User_Data
The `User_Data` can be anything. Data will just be skipped by the decoder.
https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#skippable-frames
*/
func CreateSkippableFrame(tag uint32, payload []byte) ([]byte, error) {
if len(payload) == 0 {
return nil, nil
}
if tag > 0xf {
return nil, fmt.Errorf("requested tag (%d) > 0xf", tag)
}
if len(payload) > math.MaxUint32 {
return nil, fmt.Errorf("requested skippable frame size (%d) > max uint32", len(payload))
}
dst := make([]byte, 8, len(payload)+8)
binary.LittleEndian.PutUint32(dst[0:], SkippableFrameMagic+tag)
binary.LittleEndian.PutUint32(dst[4:], uint32(len(payload)))
return append(dst, payload...), nil
}