Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[dbnode] Fix bootstrapping data for an unowned shard with commit log #2052

Merged
merged 3 commits into from
Nov 27, 2019
Merged

[dbnode] Fix bootstrapping data for an unowned shard with commit log #2052

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9cc7f21
[dbnode] Fix bootstrapping data for an unowned shard with commit log
robskillington Nov 27, 2019
cce766e
Add comment
robskillington Nov 27, 2019
440818b
Address feedback
robskillington Nov 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion src/dbnode/storage/bootstrap/bootstrapper/commitlog/source.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,7 @@ func (s *commitLogSource) Read(
ReturnMetadataAsRef: true,
}
datapointsSkippedNotBootstrappingNamespace = 0
datapointsSkippedNotBootstrappingShard = 0
startCommitLogsRead = s.nowFn()
)
s.log.Info("read commit logs start")
Expand All @@ -303,7 +304,8 @@ func (s *commitLogSource) Read(
zap.Stringer("took", s.nowFn().Sub(startCommitLogsRead)),
zap.Int("datapointsRead", datapointsRead),
zap.Int("datapointsSkippedNotInRange", datapointsSkippedNotInRange),
zap.Int("datapointsSkippedNotBootstrappingNamespace", datapointsSkippedNotBootstrappingNamespace))
zap.Int("datapointsSkippedNotBootstrappingNamespace", datapointsSkippedNotBootstrappingNamespace),
zap.Int("datapointsSkippedNotBootstrappingShard", datapointsSkippedNotBootstrappingShard))
}()

iter, corruptFiles, err := s.newIteratorFn(iterOpts)
Expand Down Expand Up @@ -476,6 +478,15 @@ func (s *commitLogSource) Read(
datapointsSkippedNotBootstrappingNamespace++
continue
}
// If not bootstrapping shard for this series then also skip.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit for future work: maybe good to have a short explanation here describing how this state can occur

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good, will do.

// NB(r): This can occur when a topology change happens then we
// bootstrap from the commit log data that the node no longer owns.
shard := seriesEntry.series.Shard
_, bootstrapping := seriesEntry.namespace.dataAndIndexShardRanges[shard]
if !bootstrapping {
datapointsSkippedNotBootstrappingShard++
continue
}

// Distribute work.
// NB(r): In future we could batch a few points together before sending
Expand Down
22 changes: 10 additions & 12 deletions src/dbnode/storage/bootstrap/process.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,9 +217,17 @@ func (b bootstrapProcess) Run(
begin := b.nowFn()
res, err := b.bootstrapper.Bootstrap(namespaces)
took := b.nowFn().Sub(begin)
if err != nil {
b.log.Error("bootstrap process error",
zap.Duration("took", took),
zap.Error(err))
return NamespaceResults{}, err
}

for _, entry := range namespaces.Namespaces.Iter() {
namespace := entry.Value()
nsID := namespace.Metadata.ID()

result, ok := res.Results.Get(nsID)
if !ok {
return NamespaceResults{},
Expand All @@ -228,11 +236,7 @@ func (b bootstrapProcess) Run(

logFields := b.logFields(namespace.Metadata, namespace.Shards,
namespace.DataTargetRange.Range, namespace.IndexTargetRange.Range)
b.logBootstrapResult(result, logFields, err, took)
}

if err != nil {
return NamespaceResults{}, err
b.logBootstrapResult(result, logFields, took)
}

bootstrapResult = MergeNamespaceResults(bootstrapResult, res)
Expand Down Expand Up @@ -286,7 +290,6 @@ func (b bootstrapProcess) logBootstrapRun(
func (b bootstrapProcess) logBootstrapResult(
result NamespaceResult,
logFields []zapcore.Field,
err error,
took time.Duration,
) {
logFields = append(logFields,
Expand All @@ -295,13 +298,8 @@ func (b bootstrapProcess) logBootstrapResult(
logFields = append(logFields,
zap.Int("numIndexBlocks", len(result.IndexResult.IndexResults())))
}
if err != nil {
logFields = append(logFields, zap.Error(err))
b.log.Info("bootstrap range completed with error", logFields...)
return
}

b.log.Info("bootstrap range completed successfully", logFields...)
b.log.Info("bootstrap range completed", logFields...)
}

func (b bootstrapProcess) targetRangesForData(
Expand Down