Skip to content

Commit

Permalink
Ensure rewrite in vlog is within transactional limits (#911)
Browse files Browse the repository at this point in the history
* Ensure rewrite in vlog is within transactional limits

With this commit the temporary list of entries built during value log
iteration is committed before it overflows transactional limits
(maxBatchSize and maxBatchCount).

Fixes #907
  • Loading branch information
Ibrahim Jarif committed Jul 4, 2019
1 parent 329b682 commit 740b849
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions value.go
Expand Up @@ -397,16 +397,19 @@ func (vlog *valueLog) rewrite(f *logFile, tr trace.Trace) error {
} }


ne.Value = append([]byte{}, e.Value...) ne.Value = append([]byte{}, e.Value...)
wb = append(wb, ne) es := int64(ne.estimateSize(vlog.opt.ValueThreshold))
size += int64(e.estimateSize(vlog.opt.ValueThreshold)) // Ensure length and size of wb is within transaction limits.
if size >= 64*mi { if int64(len(wb)+1) > vlog.opt.maxBatchCount ||
size+es > vlog.opt.maxBatchSize {
tr.LazyPrintf("request has %d entries, size %d", len(wb), size) tr.LazyPrintf("request has %d entries, size %d", len(wb), size)
if err := vlog.db.batchSet(wb); err != nil { if err := vlog.db.batchSet(wb); err != nil {
return err return err
} }
size = 0 size = 0
wb = wb[:0] wb = wb[:0]
} }
wb = append(wb, ne)
size += es
} else { } else {
vlog.db.opt.Warningf("This entry should have been caught. %+v\n", e) vlog.db.opt.Warningf("This entry should have been caught. %+v\n", e)
} }
Expand Down

0 comments on commit 740b849

Please sign in to comment.