Skip to content

Commit 398e43a

Browse files
xinhaozxinhaoz
committed
valsep: move ValueSeparation interface to new valsep pkg
This pkg will contain the components to separate values from sstables during sst writing.
1 parent dd61365 commit 398e43a

File tree

7 files changed

+142
-127
lines changed

7 files changed

+142
-127
lines changed

‎blob_rewrite_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121
"github.com/cockroachdb/pebble/internal/blobtest"
2222
"github.com/cockroachdb/pebble/internal/bytealloc"
2323
"github.com/cockroachdb/pebble/internal/cache"
24-
"github.com/cockroachdb/pebble/internal/compact"
2524
"github.com/cockroachdb/pebble/internal/manifest"
2625
"github.com/cockroachdb/pebble/internal/testkeys"
2726
"github.com/cockroachdb/pebble/internal/testutils"
@@ -30,14 +29,15 @@ import (
3029
"github.com/cockroachdb/pebble/sstable"
3130
"github.com/cockroachdb/pebble/sstable/blob"
3231
"github.com/cockroachdb/pebble/sstable/block"
32+
"github.com/cockroachdb/pebble/valsep"
3333
"github.com/cockroachdb/pebble/vfs"
3434
"github.com/stretchr/testify/require"
3535
)
3636

3737
func TestBlobRewrite(t *testing.T) {
3838
var (
3939
bv blobtest.Values
40-
vs compact.ValueSeparation
40+
vs valsep.ValueSeparation
4141
tw sstable.RawWriter
4242
fn base.DiskFileNum
4343
buf bytes.Buffer

‎compaction.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"github.com/cockroachdb/pebble/sstable/blob"
3535
"github.com/cockroachdb/pebble/sstable/block"
3636
"github.com/cockroachdb/pebble/sstable/block/blockkind"
37+
"github.com/cockroachdb/pebble/valsep"
3738
"github.com/cockroachdb/pebble/vfs"
3839
"github.com/cockroachdb/redact"
3940
)
@@ -249,7 +250,7 @@ type tableCompaction struct {
249250
// b) rewrite blob files: The compaction will write eligible values to new
250251
// blob files. This consumes more write bandwidth because all values are
251252
// rewritten. However it restores locality.
252-
getValueSeparation func(JobID, *tableCompaction, ValueStoragePolicy) compact.ValueSeparation
253+
getValueSeparation func(JobID, *tableCompaction, ValueStoragePolicy) valsep.ValueSeparation
253254

254255
// startLevel is the level that is being compacted. Inputs from startLevel
255256
// and outputLevel will be merged to produce a set of outputLevel files.
@@ -541,7 +542,7 @@ func (c *tableCompaction) makeInfo(jobID JobID) CompactionInfo {
541542
return info
542543
}
543544

544-
type getValueSeparation func(JobID, *tableCompaction, ValueStoragePolicy) compact.ValueSeparation
545+
type getValueSeparation func(JobID, *tableCompaction, ValueStoragePolicy) valsep.ValueSeparation
545546

546547
// newCompaction constructs a compaction from the provided picked compaction.
547548
//
@@ -3460,11 +3461,11 @@ func (d *DB) compactAndWrite(
34603461
vSep := valueSeparation
34613462
switch spanPolicy.ValueStoragePolicy {
34623463
case ValueStorageLowReadLatency:
3463-
vSep = compact.NeverSeparateValues{}
3464+
vSep = valsep.NeverSeparateValues{}
34643465
case ValueStorageLatencyTolerant:
34653466
// This span of keyspace is more tolerant of latency, so set a more
34663467
// aggressive value separation policy for this output.
3467-
vSep.SetNextOutputConfig(compact.ValueSeparationOutputConfig{
3468+
vSep.SetNextOutputConfig(valsep.ValueSeparationOutputConfig{
34683469
MinimumSize: latencyTolerantMinimumSize,
34693470
})
34703471
}

‎value_separation.go‎ renamed to ‎compaction_value_separation.go‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ import (
1010

1111
"github.com/cockroachdb/errors"
1212
"github.com/cockroachdb/pebble/internal/base"
13-
"github.com/cockroachdb/pebble/internal/compact"
1413
"github.com/cockroachdb/pebble/internal/invariants"
1514
"github.com/cockroachdb/pebble/internal/manifest"
1615
"github.com/cockroachdb/pebble/objstorage"
1716
"github.com/cockroachdb/pebble/sstable"
1817
"github.com/cockroachdb/pebble/sstable/blob"
18+
"github.com/cockroachdb/pebble/valsep"
1919
"github.com/cockroachdb/redact"
2020
)
2121

@@ -24,8 +24,8 @@ import (
2424
// ValueStorageLatencyTolerant.
2525
const latencyTolerantMinimumSize = 10
2626

27-
var neverSeparateValues getValueSeparation = func(JobID, *tableCompaction, ValueStoragePolicy) compact.ValueSeparation {
28-
return compact.NeverSeparateValues{}
27+
var neverSeparateValues getValueSeparation = func(JobID, *tableCompaction, ValueStoragePolicy) valsep.ValueSeparation {
28+
return valsep.NeverSeparateValues{}
2929
}
3030

3131
// determineCompactionValueSeparation determines whether a compaction should
@@ -35,22 +35,22 @@ var neverSeparateValues getValueSeparation = func(JobID, *tableCompaction, Value
3535
// It assumes that the compaction will write tables at d.TableFormat() or above.
3636
func (d *DB) determineCompactionValueSeparation(
3737
jobID JobID, c *tableCompaction, valueStorage ValueStoragePolicy,
38-
) compact.ValueSeparation {
38+
) valsep.ValueSeparation {
3939
if d.FormatMajorVersion() < FormatValueSeparation ||
4040
d.opts.Experimental.ValueSeparationPolicy == nil {
41-
return compact.NeverSeparateValues{}
41+
return valsep.NeverSeparateValues{}
4242
}
4343
policy := d.opts.Experimental.ValueSeparationPolicy()
4444
if !policy.Enabled {
45-
return compact.NeverSeparateValues{}
45+
return valsep.NeverSeparateValues{}
4646
}
4747

4848
// We're allowed to write blob references. Determine whether we should carry
4949
// forward existing blob references, or write new ones.
5050
minSize := uint64(policy.MinimumSize)
5151
switch valueStorage {
5252
case ValueStorageLowReadLatency:
53-
return compact.NeverSeparateValues{}
53+
return valsep.NeverSeparateValues{}
5454
case ValueStorageLatencyTolerant:
5555
minSize = latencyTolerantMinimumSize
5656
default:
@@ -249,10 +249,10 @@ type writeNewBlobFiles struct {
249249
}
250250

251251
// Assert that *writeNewBlobFiles implements the compact.ValueSeparation interface.
252-
var _ compact.ValueSeparation = (*writeNewBlobFiles)(nil)
252+
var _ valsep.ValueSeparation = (*writeNewBlobFiles)(nil)
253253

254254
// SetNextOutputConfig implements the ValueSeparation interface.
255-
func (vs *writeNewBlobFiles) SetNextOutputConfig(config compact.ValueSeparationOutputConfig) {
255+
func (vs *writeNewBlobFiles) SetNextOutputConfig(config valsep.ValueSeparationOutputConfig) {
256256
vs.minimumSize = config.MinimumSize
257257
}
258258

@@ -372,13 +372,13 @@ func (vs *writeNewBlobFiles) Add(
372372

373373
// FinishOutput closes the current blob file (if any). It returns the stats
374374
// and metadata of the now completed blob file.
375-
func (vs *writeNewBlobFiles) FinishOutput() (compact.ValueSeparationMetadata, error) {
375+
func (vs *writeNewBlobFiles) FinishOutput() (valsep.ValueSeparationMetadata, error) {
376376
if vs.writer == nil {
377-
return compact.ValueSeparationMetadata{}, nil
377+
return valsep.ValueSeparationMetadata{}, nil
378378
}
379379
stats, err := vs.writer.Close()
380380
if err != nil {
381-
return compact.ValueSeparationMetadata{}, err
381+
return valsep.ValueSeparationMetadata{}, err
382382
}
383383
vs.writer = nil
384384
meta := &manifest.PhysicalBlobFile{
@@ -391,7 +391,7 @@ func (vs *writeNewBlobFiles) FinishOutput() (compact.ValueSeparationMetadata, er
391391
// Reset the minimum size for the next output.
392392
vs.minimumSize = vs.globalMinimumSize
393393

394-
return compact.ValueSeparationMetadata{
394+
return valsep.ValueSeparationMetadata{
395395
BlobReferences: manifest.BlobReferences{
396396
manifest.MakeBlobReference(base.BlobFileID(vs.objMeta.DiskFileNum),
397397
stats.UncompressedValueBytes, stats.UncompressedValueBytes, meta),
@@ -442,10 +442,10 @@ type pendingReference struct {
442442

443443
// Assert that *preserveBlobReferences implements the compact.ValueSeparation
444444
// interface.
445-
var _ compact.ValueSeparation = (*preserveBlobReferences)(nil)
445+
var _ valsep.ValueSeparation = (*preserveBlobReferences)(nil)
446446

447447
// SetNextOutputConfig implements the ValueSeparation interface.
448-
func (vs *preserveBlobReferences) SetNextOutputConfig(config compact.ValueSeparationOutputConfig) {}
448+
func (vs *preserveBlobReferences) SetNextOutputConfig(config valsep.ValueSeparationOutputConfig) {}
449449

450450
// Kind implements the ValueSeparation interface.
451451
func (vs *preserveBlobReferences) Kind() sstable.ValueSeparationKind {
@@ -535,7 +535,7 @@ func (vs *preserveBlobReferences) Add(
535535
}
536536

537537
// FinishOutput implements compact.ValueSeparation.
538-
func (vs *preserveBlobReferences) FinishOutput() (compact.ValueSeparationMetadata, error) {
538+
func (vs *preserveBlobReferences) FinishOutput() (valsep.ValueSeparationMetadata, error) {
539539
if invariants.Enabled {
540540
// Assert that the incrementally-maintained totalValueSize matches the
541541
// sum of all the reference value sizes.
@@ -544,7 +544,7 @@ func (vs *preserveBlobReferences) FinishOutput() (compact.ValueSeparationMetadat
544544
totalValueSize += ref.valueSize
545545
}
546546
if totalValueSize != vs.totalValueSize {
547-
return compact.ValueSeparationMetadata{},
547+
return valsep.ValueSeparationMetadata{},
548548
errors.AssertionFailedf("totalValueSize mismatch: %d != %d", totalValueSize, vs.totalValueSize)
549549
}
550550
}
@@ -554,15 +554,15 @@ func (vs *preserveBlobReferences) FinishOutput() (compact.ValueSeparationMetadat
554554
ref := vs.currReferences[i]
555555
phys, ok := vs.inputBlobPhysicalFiles[ref.blobFileID]
556556
if !ok {
557-
return compact.ValueSeparationMetadata{},
557+
return valsep.ValueSeparationMetadata{},
558558
errors.AssertionFailedf("pebble: blob file %s not found among input sstables", ref.blobFileID)
559559
}
560560
references[i] = manifest.MakeBlobReference(ref.blobFileID, ref.valueSize, ref.valueSize, phys)
561561
}
562562
referenceSize := vs.totalValueSize
563563
vs.currReferences = vs.currReferences[:0]
564564
vs.totalValueSize = 0
565-
return compact.ValueSeparationMetadata{
565+
return valsep.ValueSeparationMetadata{
566566
BlobReferences: references,
567567
BlobReferenceSize: referenceSize,
568568
// The outputBlobReferenceDepth is computed from the input sstables,

‎value_separation_test.go‎ renamed to ‎compaction_value_separation_test.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,21 @@ import (
1818
"github.com/cockroachdb/errors"
1919
"github.com/cockroachdb/pebble/internal/base"
2020
"github.com/cockroachdb/pebble/internal/blobtest"
21-
"github.com/cockroachdb/pebble/internal/compact"
2221
"github.com/cockroachdb/pebble/internal/manifest"
2322
"github.com/cockroachdb/pebble/internal/testkeys"
2423
"github.com/cockroachdb/pebble/objstorage"
2524
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
2625
"github.com/cockroachdb/pebble/sstable"
2726
"github.com/cockroachdb/pebble/sstable/blob"
27+
"github.com/cockroachdb/pebble/valsep"
2828
"github.com/cockroachdb/pebble/vfs"
2929
"github.com/stretchr/testify/require"
3030
)
3131

3232
func TestValueSeparationPolicy(t *testing.T) {
3333
var (
3434
bv blobtest.Values
35-
vs compact.ValueSeparation
35+
vs valsep.ValueSeparation
3636
tw sstable.RawWriter
3737
fn base.DiskFileNum
3838
buf bytes.Buffer
@@ -76,7 +76,7 @@ func TestValueSeparationPolicy(t *testing.T) {
7676
bv = blobtest.Values{}
7777
switch x := d.CmdArgs[0].String(); x {
7878
case "never-separate-values":
79-
vs = compact.NeverSeparateValues{}
79+
vs = valsep.NeverSeparateValues{}
8080
case "preserve-blob-references":
8181
pbr := &preserveBlobReferences{}
8282
lines := crstrings.Lines(d.Input)
@@ -209,10 +209,10 @@ type defineDBValueSeparator struct {
209209
}
210210

211211
// Assert that *defineDBValueSeparator implements the compact.ValueSeparation interface.
212-
var _ compact.ValueSeparation = (*defineDBValueSeparator)(nil)
212+
var _ valsep.ValueSeparation = (*defineDBValueSeparator)(nil)
213213

214214
// SetNextOutputConfig implements the compact.ValueSeparation interface.
215-
func (vs *defineDBValueSeparator) SetNextOutputConfig(config compact.ValueSeparationOutputConfig) {}
215+
func (vs *defineDBValueSeparator) SetNextOutputConfig(config valsep.ValueSeparationOutputConfig) {}
216216

217217
// Kind implements the ValueSeparation interface.
218218
func (vs *defineDBValueSeparator) Kind() sstable.ValueSeparationKind {
@@ -277,10 +277,10 @@ func (vs *defineDBValueSeparator) Add(
277277
}
278278

279279
// FinishOutput implements compact.ValueSeparation.
280-
func (d *defineDBValueSeparator) FinishOutput() (compact.ValueSeparationMetadata, error) {
280+
func (d *defineDBValueSeparator) FinishOutput() (valsep.ValueSeparationMetadata, error) {
281281
m, err := d.pbr.FinishOutput()
282282
if err != nil {
283-
return compact.ValueSeparationMetadata{}, err
283+
return valsep.ValueSeparationMetadata{}, err
284284
}
285285
// TODO(jackson): Support setting a specific depth from the datadriven test.
286286
m.BlobReferenceDepth = manifest.BlobReferenceDepth(len(m.BlobReferences))

‎data_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626
"github.com/cockroachdb/pebble/bloom"
2727
"github.com/cockroachdb/pebble/internal/base"
2828
"github.com/cockroachdb/pebble/internal/blobtest"
29-
"github.com/cockroachdb/pebble/internal/compact"
3029
"github.com/cockroachdb/pebble/internal/humanize"
3130
"github.com/cockroachdb/pebble/internal/keyspan"
3231
"github.com/cockroachdb/pebble/internal/manifest"
@@ -39,6 +38,7 @@ import (
3938
"github.com/cockroachdb/pebble/objstorage/remote"
4039
"github.com/cockroachdb/pebble/sstable"
4140
"github.com/cockroachdb/pebble/sstable/block/blockkind"
41+
"github.com/cockroachdb/pebble/valsep"
4242
"github.com/cockroachdb/pebble/vfs"
4343
"github.com/cockroachdb/pebble/vfs/errorfs"
4444
"github.com/cockroachdb/pebble/wal"
@@ -939,7 +939,7 @@ func runDBDefineCmd(td *datadriven.TestData, opts *Options) (*DB, error) {
939939
if err != nil {
940940
return err
941941
}
942-
c.getValueSeparation = func(JobID, *tableCompaction, ValueStoragePolicy) compact.ValueSeparation {
942+
c.getValueSeparation = func(JobID, *tableCompaction, ValueStoragePolicy) valsep.ValueSeparation {
943943
return valueSeparator
944944
}
945945
// NB: define allows the test to exactly specify which keys go

0 commit comments

Comments
 (0)