Skip to content

Commit e494c55

Browse files
xinhaozxinhaoz
committed
objstorage: move ReadableFile and SimpleReadable form sstable -> objstorage
Allow opening blob files as SimpleReadable.
1 parent de16e15 commit e494c55

23 files changed

+112
-98
lines changed

‎data_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1359,7 +1359,7 @@ func runSSTablePropertiesCmd(t *testing.T, td *datadriven.TestData, d *DB) strin
13591359
if err != nil {
13601360
return err.Error()
13611361
}
1362-
readable, err := sstable.NewSimpleReadable(f)
1362+
readable, err := objstorage.NewSimpleReadable(f)
13631363
if err != nil {
13641364
return err.Error()
13651365
}
@@ -1411,7 +1411,7 @@ func runLayoutCmd(t *testing.T, td *datadriven.TestData, d *DB) string {
14111411
if err != nil {
14121412
return err.Error()
14131413
}
1414-
readable, err := sstable.NewSimpleReadable(f)
1414+
readable, err := objstorage.NewSimpleReadable(f)
14151415
if err != nil {
14161416
return err.Error()
14171417
}

‎excise_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/cockroachdb/pebble/internal/manifest"
2222
"github.com/cockroachdb/pebble/internal/testkeys"
2323
"github.com/cockroachdb/pebble/internal/testutils"
24+
"github.com/cockroachdb/pebble/objstorage"
2425
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
2526
"github.com/cockroachdb/pebble/objstorage/remote"
2627
"github.com/cockroachdb/pebble/sstable"
@@ -827,7 +828,7 @@ func TestExciseBounds(t *testing.T) {
827828
case "excise":
828829
f, err := fs.Open(sstPath)
829830
checkErr(err)
830-
readable, err := sstable.NewSimpleReadable(f)
831+
readable, err := objstorage.NewSimpleReadable(f)
831832
checkErr(err)
832833
ctx := context.Background()
833834
r, err := sstable.NewReader(ctx, readable, sstable.ReaderOptions{})

‎external_iterator.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/cockroachdb/pebble/internal/base"
1212
"github.com/cockroachdb/pebble/internal/keyspan"
1313
"github.com/cockroachdb/pebble/internal/manifest"
14+
"github.com/cockroachdb/pebble/objstorage"
1415
"github.com/cockroachdb/pebble/sstable"
1516
"github.com/cockroachdb/pebble/sstable/block"
1617
)
@@ -33,15 +34,15 @@ import (
3334
// options, including block-property and table filters. NewExternalIter errors
3435
// if an incompatible option is set.
3536
func NewExternalIter(
36-
o *Options, iterOpts *IterOptions, files [][]sstable.ReadableFile,
37+
o *Options, iterOpts *IterOptions, files [][]objstorage.ReadableFile,
3738
) (it *Iterator, err error) {
3839
return NewExternalIterWithContext(context.Background(), o, iterOpts, files)
3940
}
4041

4142
// NewExternalIterWithContext is like NewExternalIter, and additionally
4243
// accepts a context for tracing.
4344
func NewExternalIterWithContext(
44-
ctx context.Context, o *Options, iterOpts *IterOptions, files [][]sstable.ReadableFile,
45+
ctx context.Context, o *Options, iterOpts *IterOptions, files [][]objstorage.ReadableFile,
4546
) (it *Iterator, err error) {
4647
if iterOpts != nil {
4748
if err := validateExternalIterOpts(iterOpts); err != nil {
@@ -297,11 +298,11 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) error {
297298
}
298299

299300
func openExternalTables(
300-
ctx context.Context, files []sstable.ReadableFile, readerOpts sstable.ReaderOptions,
301+
ctx context.Context, files []objstorage.ReadableFile, readerOpts sstable.ReaderOptions,
301302
) (readers []*sstable.Reader, err error) {
302303
readers = make([]*sstable.Reader, 0, len(files))
303304
for i := range files {
304-
readable, err := sstable.NewSimpleReadable(files[i])
305+
readable, err := objstorage.NewSimpleReadable(files[i])
305306
if err != nil {
306307
return readers, err
307308
}

‎external_iterator_test.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/cockroachdb/errors"
1616
"github.com/cockroachdb/pebble/internal/blobtest"
1717
"github.com/cockroachdb/pebble/internal/testkeys"
18+
"github.com/cockroachdb/pebble/objstorage"
1819
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
1920
"github.com/cockroachdb/pebble/sstable"
2021
"github.com/cockroachdb/pebble/vfs"
@@ -35,7 +36,7 @@ func TestExternalIterator(t *testing.T) {
3536
defer func() { require.NoError(t, d.Close()) }()
3637
var bv blobtest.Values
3738

38-
getOptsAndFiles := func(td *datadriven.TestData) (opts IterOptions, files [][]sstable.ReadableFile) {
39+
getOptsAndFiles := func(td *datadriven.TestData) (opts IterOptions, files [][]objstorage.ReadableFile) {
3940
opts = IterOptions{KeyTypes: IterKeyTypePointsAndRanges}
4041
for _, arg := range td.CmdArgs {
4142
switch arg.Key {
@@ -49,7 +50,7 @@ func TestExternalIterator(t *testing.T) {
4950
for _, v := range arg.Vals {
5051
f, err := mem.Open(v)
5152
require.NoError(t, err)
52-
files = append(files, []sstable.ReadableFile{f})
53+
files = append(files, []objstorage.ReadableFile{f})
5354
}
5455
}
5556
}
@@ -89,7 +90,7 @@ func TestExternalIterator(t *testing.T) {
8990
// See github.com/cockroachdb/cockroach/issues/141606 where an error during
9091
// initialization caused NewExternalIter to panic.
9192
func testExternalIteratorInitError(
92-
t *testing.T, o *Options, iterOpts *IterOptions, files [][]sstable.ReadableFile,
93+
t *testing.T, o *Options, iterOpts *IterOptions, files [][]objstorage.ReadableFile,
9394
) {
9495
files = slices.Clone(files)
9596
for i := range files {
@@ -110,7 +111,7 @@ func testExternalIteratorInitError(
110111
}
111112

112113
type flakyFile struct {
113-
sstable.ReadableFile
114+
objstorage.ReadableFile
114115
}
115116

116117
func (ff *flakyFile) ReadAt(p []byte, off int64) (n int, err error) {
@@ -163,11 +164,11 @@ func BenchmarkExternalIter_NonOverlapping_Scan(b *testing.B) {
163164
b.ResetTimer()
164165
for i := 0; i < b.N; i++ {
165166
func() {
166-
files := make([][]sstable.ReadableFile, fileCount)
167+
files := make([][]objstorage.ReadableFile, fileCount)
167168
for i := 0; i < fileCount; i++ {
168169
f, err := fs.Open(filenames[i])
169170
require.NoError(b, err)
170-
files[i] = []sstable.ReadableFile{f}
171+
files[i] = []objstorage.ReadableFile{f}
171172
}
172173

173174
it, err := NewExternalIter(opts, iterOpts, files)

‎ingest.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ func ingestLoad(
524524
return ingestLoadResult{}, err
525525
}
526526

527-
readable, err := sstable.NewSimpleReadable(f)
527+
readable, err := objstorage.NewSimpleReadable(f)
528528
if err != nil {
529529
return ingestLoadResult{}, err
530530
}

‎ingest_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2972,7 +2972,7 @@ func TestIngestValidation(t *testing.T) {
29722972
defer func() { require.NoError(t, d.Close()) }()
29732973

29742974
corrupt := func(f vfs.File) {
2975-
readable, err := sstable.NewSimpleReadable(f)
2975+
readable, err := objstorage.NewSimpleReadable(f)
29762976
require.NoError(t, err)
29772977
// Compute the layout of the sstable in order to find the
29782978
// appropriate block locations to corrupt.

‎level_checker_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/cockroachdb/pebble/internal/sstableinternal"
2525
"github.com/cockroachdb/pebble/internal/testkeys"
2626
"github.com/cockroachdb/pebble/internal/testutils"
27+
"github.com/cockroachdb/pebble/objstorage"
2728
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
2829
"github.com/cockroachdb/pebble/sstable"
2930
"github.com/cockroachdb/pebble/sstable/blob"
@@ -230,7 +231,7 @@ func TestCheckLevelsCornerCases(t *testing.T) {
230231
if err != nil {
231232
return err.Error()
232233
}
233-
readable, err := sstable.NewSimpleReadable(f)
234+
readable, err := objstorage.NewSimpleReadable(f)
234235
if err != nil {
235236
return err.Error()
236237
}

‎level_iter_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/cockroachdb/pebble/internal/rangekey"
2626
"github.com/cockroachdb/pebble/internal/sstableinternal"
2727
"github.com/cockroachdb/pebble/internal/testkeys"
28+
"github.com/cockroachdb/pebble/objstorage"
2829
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
2930
"github.com/cockroachdb/pebble/sstable"
3031
"github.com/cockroachdb/pebble/sstable/block"
@@ -291,7 +292,7 @@ func (lt *levelIterTest) runBuild(d *datadriven.TestData) string {
291292
if err != nil {
292293
return err.Error()
293294
}
294-
readable, err := sstable.NewSimpleReadable(f1)
295+
readable, err := objstorage.NewSimpleReadable(f1)
295296
if err != nil {
296297
return err.Error()
297298
}
@@ -561,7 +562,7 @@ func buildLevelIterTables(
561562
if err != nil {
562563
b.Fatal(err)
563564
}
564-
readable, err := sstable.NewSimpleReadable(f)
565+
readable, err := objstorage.NewSimpleReadable(f)
565566
if err != nil {
566567
b.Fatal(err)
567568
}

‎merging_iter_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/cockroachdb/pebble/internal/sstableinternal"
2828
"github.com/cockroachdb/pebble/internal/testkeys"
2929
"github.com/cockroachdb/pebble/internal/testutils/indenttree"
30+
"github.com/cockroachdb/pebble/objstorage"
3031
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
3132
"github.com/cockroachdb/pebble/sstable"
3233
"github.com/cockroachdb/pebble/sstable/block"
@@ -265,7 +266,7 @@ func TestMergingIterDataDriven(t *testing.T) {
265266
if err != nil {
266267
return err.Error()
267268
}
268-
readable, err := sstable.NewSimpleReadable(f)
269+
readable, err := objstorage.NewSimpleReadable(f)
269270
if err != nil {
270271
return err.Error()
271272
}
@@ -392,7 +393,7 @@ func buildMergingIterTables(
392393
if err != nil {
393394
b.Fatal(err)
394395
}
395-
readable, err := sstable.NewSimpleReadable(f)
396+
readable, err := objstorage.NewSimpleReadable(f)
396397
if err != nil {
397398
b.Fatal(err)
398399
}
@@ -657,7 +658,7 @@ func buildLevelsForMergingIterSeqSeek(
657658
if err != nil {
658659
b.Fatal(err)
659660
}
660-
readable, err := sstable.NewSimpleReadable(f)
661+
readable, err := objstorage.NewSimpleReadable(f)
661662
if err != nil {
662663
b.Fatal(err)
663664
}

‎metamorphic/options_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/cockroachdb/errors"
2020
"github.com/cockroachdb/pebble"
2121
"github.com/cockroachdb/pebble/internal/testkeys"
22+
"github.com/cockroachdb/pebble/objstorage"
2223
"github.com/cockroachdb/pebble/sstable"
2324
"github.com/cockroachdb/pebble/vfs"
2425
"github.com/kr/pretty"
@@ -231,7 +232,7 @@ func TestBlockPropertiesParse(t *testing.T) {
231232
if err != nil {
232233
return err
233234
}
234-
readable, err := sstable.NewSimpleReadable(f)
235+
readable, err := objstorage.NewSimpleReadable(f)
235236
if err != nil {
236237
return err
237238
}

0 commit comments

Comments
 (0)