Skip to content

Commit

Permalink
storageccl: Add resume key timestamp to export req/res.
Browse files Browse the repository at this point in the history
Add ability to request exports that would use resume span starting
from arbitrary key timestamp and return resume timestamp for
continuation to use new storage export capability.

Release note: None
  • Loading branch information
aliher1911 committed Aug 6, 2021
1 parent 2082744 commit 7f6fb0a
Show file tree
Hide file tree
Showing 4 changed files with 736 additions and 557 deletions.
14 changes: 12 additions & 2 deletions pkg/ccl/storageccl/export.go
Expand Up @@ -178,11 +178,19 @@ func evalExport(

// Time-bound iterators only make sense to use if the start time is set.
useTBI := args.EnableTimeBoundIteratorOptimization && !args.StartTime.IsEmpty()
// Only use resume timestamp if splitting mid key is enabled.
resumeKeyTS := hlc.Timestamp{}
if args.SplitMidKey {
if !args.ReturnSST {
return result.Result{}, errors.New("SplitMidKey could only be used with ReturnSST option")
}
resumeKeyTS = args.ResumeKeyTS
}
var curSizeOfExportedSSTs int64
for start := args.Key; start != nil; {
destFile := &storage.MemFile{}
summary, resume, _, err := reader.ExportMVCCToSst(ctx, start, args.EndKey, args.StartTime,
h.Timestamp, hlc.Timestamp{}, exportAllRevisions, targetSize, maxSize, false, useTBI, destFile)
summary, resume, resumeTS, err := reader.ExportMVCCToSst(ctx, start, args.EndKey, args.StartTime,
h.Timestamp, resumeKeyTS, exportAllRevisions, targetSize, maxSize, args.SplitMidKey, useTBI, destFile)
if err != nil {
if errors.HasType(err, (*storage.ExceedMaxSizeError)(nil)) {
err = errors.WithHintf(err,
Expand All @@ -207,6 +215,7 @@ func evalExport(
}
exported := roachpb.ExportResponse_File{
Span: span,
EndKeyTS: resumeTS,
Exported: summary,
LocalityKV: localityKV,
}
Expand Down Expand Up @@ -250,6 +259,7 @@ func evalExport(
}
reply.Files = append(reply.Files, exported)
start = resume
resumeKeyTS = resumeTS

// If we are not returning the SSTs to the processor, there is no need to
// paginate the ExportRequest since the reply size will not grow large
Expand Down

0 comments on commit 7f6fb0a

Please sign in to comment.