Skip to content

Commit 34e929e

Browse files
jbowensjbowens
committed
sstable: clean up datadriven build commands
Clean up the datadriven build and build-raw commands uses by various sstable unit tests. The build command confusingly had its own fragmenters for range deletions and range keys sitting between the test instructions and the Writer being tested. These commands are updated so that the build instructions mirror what is supplied to the writer.
1 parent 6637383 commit 34e929e

File tree

8 files changed

+100
-148
lines changed

8 files changed

+100
-148
lines changed

‎sstable/data_test.go‎

Lines changed: 34 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -68,105 +68,55 @@ func optsFromArgs(td *datadriven.TestData, writerOpts *WriterOptions) error {
6868
func runBuildCmd(
6969
td *datadriven.TestData, writerOpts *WriterOptions, cacheSize int,
7070
) (*WriterMetadata, *Reader, error) {
71-
7271
f0 := &objstorage.MemObj{}
7372
if err := optsFromArgs(td, writerOpts); err != nil {
7473
return nil, nil, err
7574
}
7675

77-
w := NewWriter(f0, *writerOpts)
76+
w := NewRawWriter(f0, *writerOpts)
7877
defer func() {
7978
if w != nil {
8079
_ = w.Close()
8180
}
8281
}()
83-
var rangeDels []keyspan.Span
84-
rangeDelFrag := keyspan.Fragmenter{
85-
Cmp: DefaultComparer.Compare,
86-
Format: DefaultComparer.FormatKey,
87-
Emit: func(s keyspan.Span) {
88-
rangeDels = append(rangeDels, s)
89-
},
90-
}
91-
var rangeKeys []keyspan.Span
92-
rangeKeyFrag := keyspan.Fragmenter{
93-
Cmp: DefaultComparer.Compare,
94-
Format: DefaultComparer.FormatKey,
95-
Emit: func(s keyspan.Span) {
96-
rangeKeys = append(rangeKeys, s)
97-
},
98-
}
9982
for _, data := range strings.Split(td.Input, "\n") {
100-
if strings.HasPrefix(data, "rangekey:") {
101-
var err error
102-
func() {
103-
defer func() {
104-
if r := recover(); r != nil {
105-
err = errors.Errorf("%v", r)
106-
}
107-
}()
108-
rangeKeyFrag.Add(keyspan.ParseSpan(strings.TrimPrefix(data, "rangekey:")))
83+
err := func() (err error) {
84+
defer func() {
85+
if r := recover(); r != nil {
86+
err = errors.Errorf("%v", r)
87+
}
10988
}()
110-
if err != nil {
111-
return nil, nil, err
112-
}
113-
continue
114-
}
11589

116-
forceObsolete := false
117-
if strings.HasPrefix(data, "force-obsolete:") {
118-
data = strings.TrimSpace(strings.TrimPrefix(data, "force-obsolete:"))
119-
forceObsolete = true
120-
}
121-
j := strings.Index(data, ":")
122-
key := base.ParseInternalKey(data[:j])
123-
value := []byte(data[j+1:])
124-
switch key.Kind() {
125-
case InternalKeyKindRangeDelete:
126-
if forceObsolete {
127-
return nil, nil, errors.Errorf("force-obsolete is not allowed for RANGEDEL")
128-
}
129-
var err error
130-
func() {
131-
defer func() {
132-
if r := recover(); r != nil {
133-
err = errors.Errorf("%v", r)
90+
switch {
91+
case strings.HasPrefix(data, "EncodeSpan:"):
92+
return w.EncodeSpan(keyspan.ParseSpan(strings.TrimPrefix(data, "EncodeSpan:")))
93+
default:
94+
forceObsolete := strings.HasPrefix(data, "force-obsolete:")
95+
if forceObsolete {
96+
data = strings.TrimSpace(strings.TrimPrefix(data, "force-obsolete:"))
97+
}
98+
j := strings.Index(data, ":")
99+
key := base.ParseInternalKey(data[:j])
100+
value := []byte(data[j+1:])
101+
switch key.Kind() {
102+
case InternalKeyKindRangeDelete:
103+
if forceObsolete {
104+
return errors.Errorf("force-obsolete is not allowed for RANGEDEL")
134105
}
135-
}()
136-
rangeDelFrag.Add(keyspan.Span{
137-
Start: key.UserKey,
138-
End: value,
139-
Keys: []keyspan.Key{{Trailer: key.Trailer}},
140-
})
141-
}()
142-
if err != nil {
143-
return nil, nil, err
144-
}
145-
default:
146-
if err := w.Raw().AddWithForceObsolete(key, value, forceObsolete); err != nil {
147-
return nil, nil, err
148-
}
149-
}
150-
}
151-
rangeDelFrag.Finish()
152-
for _, v := range rangeDels {
153-
for _, k := range v.Keys {
154-
ik := base.InternalKey{UserKey: v.Start, Trailer: k.Trailer}
155-
if err := w.Raw().AddWithForceObsolete(ik, v.End, false); err != nil {
156-
return nil, nil, err
106+
return w.AddWithForceObsolete(key, value, false /* forceObsolete */)
107+
default:
108+
return w.AddWithForceObsolete(key, value, forceObsolete)
109+
}
157110
}
158-
}
159-
}
160-
rangeKeyFrag.Finish()
161-
for _, s := range rangeKeys {
162-
if err := w.addRangeKeySpanToFragmenter(s); err != nil {
111+
}()
112+
if err != nil {
163113
return nil, nil, err
164114
}
165115
}
166116
if err := w.Close(); err != nil {
167117
return nil, nil, err
168118
}
169-
meta, err := w.Raw().Metadata()
119+
meta, err := w.Metadata()
170120
w = nil
171121
if err != nil {
172122
return nil, nil, err
@@ -209,16 +159,16 @@ func runBuildRawCmd(
209159
return nil, nil, err
210160
}
211161

212-
w := NewWriter(f0, *opts)
162+
w := NewRawWriter(f0, *opts)
213163
defer func() {
214164
if w != nil {
215165
_ = w.Close()
216166
}
217167
}()
218168
for _, data := range strings.Split(td.Input, "\n") {
219-
if strings.HasPrefix(data, "rangekey:") {
220-
data = strings.TrimPrefix(data, "rangekey:")
221-
if err := w.addRangeKeySpanToFragmenter(keyspan.ParseSpan(data)); err != nil {
169+
if strings.HasPrefix(data, "EncodeSpan:") {
170+
data = strings.TrimPrefix(data, "EncodeSpan:")
171+
if err := w.EncodeSpan(keyspan.ParseSpan(data)); err != nil {
222172
return nil, nil, err
223173
}
224174
continue
@@ -227,14 +177,14 @@ func runBuildRawCmd(
227177
j := strings.Index(data, ":")
228178
key := base.ParseInternalKey(data[:j])
229179
value := []byte(data[j+1:])
230-
if err := w.Raw().AddWithForceObsolete(key, value, false); err != nil {
180+
if err := w.AddWithForceObsolete(key, value, false); err != nil {
231181
return nil, nil, err
232182
}
233183
}
234184
if err := w.Close(); err != nil {
235185
return nil, nil, err
236186
}
237-
meta, err := w.Raw().Metadata()
187+
meta, err := w.Metadata()
238188
w = nil
239189
if err != nil {
240190
return nil, nil, err

‎sstable/testdata/block_properties‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,9 @@ build collectors=(suffix)
108108
a@5.SET.1:foo
109109
b@10.SET.2:bar
110110
c@15.SET.3:baz
111-
rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
112-
rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
113-
rangekey: f@20-z@25:{(#6,RANGEKEYDEL)}
111+
EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
112+
EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
113+
EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)}
114114
----
115115
point: [a@5#1,SET-c@15#3,SET]
116116
rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL]
@@ -135,9 +135,9 @@ build collectors=(suffix-point-keys-only)
135135
a@5.SET.1:foo
136136
b@10.SET.2:bar
137137
c@15.SET.3:baz
138-
rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
139-
rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
140-
rangekey: f@20-z@25:{(#6,RANGEKEYDEL)}
138+
EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
139+
EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
140+
EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)}
141141
----
142142
point: [a@5#1,SET-c@15#3,SET]
143143
rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL]
@@ -162,9 +162,9 @@ build collectors=(suffix-range-keys-only)
162162
a@5.SET.1:foo
163163
b@10.SET.2:bar
164164
c@15.SET.3:baz
165-
rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
166-
rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
167-
rangekey: f@20-z@25:{(#6,RANGEKEYDEL)}
165+
EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
166+
EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
167+
EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)}
168168
----
169169
point: [a@5#1,SET-c@15#3,SET]
170170
rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL]
@@ -190,9 +190,9 @@ build block-size=1 collectors=(suffix-point-keys-only,suffix-range-keys-only)
190190
a@5.SET.1:foo
191191
b@10.SET.2:bar
192192
c@15.SET.3:baz
193-
rangekey: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
194-
rangekey: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
195-
rangekey: f@20-z@25:{(#6,RANGEKEYDEL)}
193+
EncodeSpan: d@10-e@15:{(#4,RANGEKEYSET,@20,foo)}
194+
EncodeSpan: e@15-f@20:{(#5,RANGEKEYUNSET,@25)}
195+
EncodeSpan: f@20-z@25:{(#6,RANGEKEYDEL)}
196196
----
197197
point: [a@5#1,SET-c@15#3,SET]
198198
rangekey: [d@10#4,RANGEKEYSET-z@25#inf,RANGEKEYDEL]

‎sstable/testdata/rewriter‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -212,9 +212,9 @@ c
212212
# Rewrite a table that contain only range keys.
213213

214214
build block-size=1 index-block-size=1 filter comparer-split-4b-suffix
215-
rangekey: a-b:{(#1,RANGEKEYSET,_xyz)}
216-
rangekey: b-c:{(#1,RANGEKEYSET,_xyz)}
217-
rangekey: c-d:{(#1,RANGEKEYSET,_xyz)}
215+
EncodeSpan: a-b:{(#1,RANGEKEYSET,_xyz)}
216+
EncodeSpan: b-c:{(#1,RANGEKEYSET,_xyz)}
217+
EncodeSpan: c-d:{(#1,RANGEKEYSET,_xyz)}
218218
----
219219
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
220220
seqnums: [1-1]

‎sstable/testdata/rewriter_v3‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -212,9 +212,9 @@ c
212212
# Rewrite a table that contain only range keys.
213213

214214
build block-size=1 index-block-size=1 filter comparer-split-4b-suffix
215-
rangekey: a-b:{(#1,RANGEKEYSET,_xyz)}
216-
rangekey: b-c:{(#1,RANGEKEYSET,_xyz)}
217-
rangekey: c-d:{(#1,RANGEKEYSET,_xyz)}
215+
EncodeSpan: a-b:{(#1,RANGEKEYSET,_xyz)}
216+
EncodeSpan: b-c:{(#1,RANGEKEYSET,_xyz)}
217+
EncodeSpan: c-d:{(#1,RANGEKEYSET,_xyz)}
218218
----
219219
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
220220
seqnums: [1-1]

‎sstable/testdata/virtual_reader_iter‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ a.SET.1:a
3232
d.SET.2:d
3333
f.SET.3:f
3434
d.RANGEDEL.4:e
35-
rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)}
35+
EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)}
3636
g.RANGEDEL.5:l
37-
rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)}
37+
EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)}
3838
----
3939
point: [a#1,SET-f#3,SET]
4040
rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL]
@@ -569,9 +569,9 @@ b.SET.5:b
569569
d.SET.2:d
570570
f.SET.3:f
571571
d.RANGEDEL.4:e
572-
rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)}
572+
EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)}
573573
g.RANGEDEL.5:l
574-
rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)}
574+
EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)}
575575
----
576576
point: [a#1,SET-f#3,SET]
577577
rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL]

‎sstable/testdata/virtual_reader_props‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,9 @@ a.SET.1:a
101101
d.SET.2:d
102102
f.SET.3:f
103103
d.RANGEDEL.4:e
104-
rangekey: a-d:{(#11,RANGEKEYSET,@10,foo)}
104+
EncodeSpan: a-d:{(#11,RANGEKEYSET,@10,foo)}
105105
g.RANGEDEL.5:l
106-
rangekey: y-z:{(#12,RANGEKEYSET,@11,foo)}
106+
EncodeSpan: y-z:{(#12,RANGEKEYSET,@11,foo)}
107107
----
108108
point: [a#1,SET-f#3,SET]
109109
rangedel: [d#4,RANGEDEL-l#inf,RANGEDEL]

‎sstable/testdata/writer‎

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ f.SET.5:f
2323
g.DEL.6:
2424
h.MERGE.7:h
2525
i.RANGEDEL.8:j
26-
rangekey: j-k:{(#9,RANGEKEYDEL)}
27-
rangekey: k-l:{(#10,RANGEKEYUNSET,@5)}
28-
rangekey: l-m:{(#11,RANGEKEYSET,@10,foo)}
26+
EncodeSpan: j-k:{(#9,RANGEKEYDEL)}
27+
EncodeSpan: k-l:{(#10,RANGEKEYUNSET,@5)}
28+
EncodeSpan: l-m:{(#11,RANGEKEYSET,@10,foo)}
2929
----
3030
point: [a#1,SET-h#7,MERGE]
3131
rangedel: [d#4,RANGEDEL-j#inf,RANGEDEL]
@@ -46,9 +46,9 @@ e.SINGLEDEL.5:
4646
f.SET.6:f
4747
g.DEL.7:
4848
h.SINGLEDEL.8:
49-
rangekey: j-k:{(#9,RANGEKEYDEL)}
50-
rangekey: k-l:{(#10,RANGEKEYUNSET,@5)}
51-
rangekey: l-m:{(#11,RANGEKEYSET,@10,foo)}
49+
EncodeSpan: j-k:{(#9,RANGEKEYDEL)}
50+
EncodeSpan: k-l:{(#10,RANGEKEYUNSET,@5)}
51+
EncodeSpan: l-m:{(#11,RANGEKEYSET,@10,foo)}
5252
----
5353
point: [a#1,SET-h#8,SINGLEDEL]
5454
rangekey: [j#9,RANGEKEYDEL-m#inf,RANGEKEYSET]
@@ -91,9 +91,11 @@ i-j:{(#8,RANGEDEL)}
9191
# 1: j---------------z
9292

9393
build
94-
a.RANGEDEL.3:m
95-
f.RANGEDEL.2:s
96-
j.RANGEDEL.1:z
94+
EncodeSpan: a-f:{(#3,RANGEDEL)}
95+
EncodeSpan: f-j:{(#3,RANGEDEL) (#2,RANGEDEL)}
96+
EncodeSpan: j-m:{(#3,RANGEDEL) (#2,RANGEDEL) (#1,RANGEDEL)}
97+
EncodeSpan: m-s:{(#2,RANGEDEL) (#1,RANGEDEL)}
98+
EncodeSpan: s-z:{(#1,RANGEDEL)}
9799
----
98100
rangedel: [a#3,RANGEDEL-z#inf,RANGEDEL]
99101
seqnums: [1-3]
@@ -157,7 +159,7 @@ build
157159
b.RANGEDEL.1:c
158160
a.RANGEDEL.2:b
159161
----
160-
pebble: keys must be added in order: b > a
162+
pebble: keys must be added in order: b#1,RANGEDEL, a#2,RANGEDEL
161163

162164
build-raw
163165
.RANGEDEL.1:b
@@ -190,22 +192,21 @@ c.RANGEDEL.2:d
190192
rangedel: [a#1,RANGEDEL-d#inf,RANGEDEL]
191193
seqnums: [1-2]
192194

193-
build-raw
194-
rangekey: a-b:{(#1,RANGEKEYSET,@10,foo)}
195-
rangekey: a-b:{(#2,RANGEKEYSET,@10,foo)}
195+
build
196+
EncodeSpan: a-b:{(#2,RANGEKEYSET,@10,foo) (#1,RANGEKEYSET,@10,foo)}
196197
----
197198
rangekey: [a#2,RANGEKEYSET-b#inf,RANGEKEYSET]
198199
seqnums: [1-2]
199200

200201
build-raw
201-
rangekey: b-c:{(#2,RANGEKEYSET,@10,foo)}
202-
rangekey: a-b:{(#1,RANGEKEYSET,@10,foo)}
202+
EncodeSpan: b-c:{(#2,RANGEKEYSET,@10,foo)}
203+
EncodeSpan: a-b:{(#1,RANGEKEYSET,@10,foo)}
203204
----
204-
pebble: spans must be added in order: b > a
205+
pebble: range keys starts must be added in increasing order: b#2,RANGEKEYSET, a#1,RANGEKEYSET
205206

206207
build-raw
207-
rangekey: a-c:{(#1,RANGEKEYSET,@10,foo)}
208-
rangekey: c-d:{(#2,RANGEKEYSET,@10,foo)}
208+
EncodeSpan: a-c:{(#1,RANGEKEYSET,@10,foo)}
209+
EncodeSpan: c-d:{(#2,RANGEKEYSET,@10,foo)}
209210
----
210211
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
211212
seqnums: [1-2]
@@ -214,7 +215,7 @@ seqnums: [1-2]
214215
# though the key kinds must be ordered (descending).
215216

216217
build-raw
217-
rangekey: a-b:{(#1,RANGEKEYSET,@10,foo) (#1,RANGEKEYUNSET,@t10) (#1,RANGEKEYDEL)}
218+
EncodeSpan: a-b:{(#1,RANGEKEYSET,@10,foo) (#1,RANGEKEYUNSET,@t10) (#1,RANGEKEYDEL)}
218219
----
219220
rangekey: [a#1,RANGEKEYSET-b#inf,RANGEKEYDEL]
220221
seqnums: [1-1]
@@ -274,9 +275,9 @@ layout
274275
# Range keys, if present, are shown in the layout.
275276

276277
build
277-
rangekey: a-b:{(#3,RANGEKEYSET,@3,foo)}
278-
rangekey: b-c:{(#2,RANGEKEYSET,@2,bar)}
279-
rangekey: c-d:{(#1,RANGEKEYSET,@1,baz)}
278+
EncodeSpan: a-b:{(#3,RANGEKEYSET,@3,foo)}
279+
EncodeSpan: b-c:{(#2,RANGEKEYSET,@2,bar)}
280+
EncodeSpan: c-d:{(#1,RANGEKEYSET,@1,baz)}
280281
----
281282
rangekey: [a#3,RANGEKEYSET-d#inf,RANGEKEYSET]
282283
seqnums: [1-3]

0 commit comments

Comments
 (0)