Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Mv level work2 #76

Merged
merged 2 commits into from

2 participants

@matthewvon

This branch provides further tuning to the mv-level-work1 branch. The tuning is focused upon reducing when write throttle penalties apply and the performance between levels 2 and 3. The level 2 to 3 transition is when the code shifts from overlapped .sst files to sorted. The change slightly increases the write amplification (bad) but reduces latencies, both overall and worst case (good).

MatthewVon added some commits
MatthewVon Heavy rework of penalty factors. These are close, but likely one more…
… adjustment. Mostly helps worst case latencies, not overall throughput.
1940c39
matthewvon tune level 3 to have size approximate to what level 2 will dump into …
…it. Do not start penalties on sorted levels until 2.5 times limit.
730beb8
@matthewvon matthewvon was assigned
@engelsanchez
Collaborator

I've reviewed the code enough to understand what is going on and visually detect any fat fingering and such.

Besides the new tuning magic numbers, we have:

  • Penalties not triggering so soon, especially when just reaching # file compaction threshold
  • The previous changes to allow levels beyond 0 to have overlapping files was not been completely effective because the code in Compaction:ShouldStopBefore was triggering and smaller compaction files were generated, which ended up hurting performance when compaction through the first non-overlapping level occurred

:+1: :dancer:

@matthewvon matthewvon merged commit b4b2d27 into from
@engelsanchez engelsanchez was assigned
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Apr 2, 2013
  1. Heavy rework of penalty factors. These are close, but likely one more…

    MatthewVon authored
    … adjustment. Mostly helps worst case latencies, not overall throughput.
Commits on Apr 4, 2013
  1. tune level 3 to have size approximate to what level 2 will dump into …

    matthewvon authored
    …it. Do not start penalties on sorted levels until 2.5 times limit.
This page is out of date. Refresh to see the latest.
Showing with 34 additions and 62 deletions.
  1. +34 −62 db/version_set.cc
View
96 db/version_set.cc
@@ -21,7 +21,6 @@
namespace leveldb {
-#if 1
// branch mv-level-work1, March 2013
//
// Notes:
@@ -47,52 +46,17 @@ static struct
// being written to level-2. The value is five times the 300,000,000 of level-1.
{
- {10485760, 262144000, 576716800, 209715200, 300000000, true},
- {10485760, 262144000, 576716800, 419430400,1500000000, true},
- {10485760, 262144000, 576716800, 4194304000, 314572800, true},
- {10485760, 262144000, 576716800, 41943040000, 419430400, false},
- {10485760, 262144000, 576716800, 419430400000, 524288000, false},
- {10485760, 262144000, 576716800, 4194304000000, 629145600, false},
- {10485760, 262144000, 576716800, 41943040000000, 734003200, false}
+ {10485760, 262144000, 576716800, 209715200, 300000000, true},
+ {10485760, 262144000, 576716800, 419430400,1500000000, true},
+ {10485760, 262144000, 576716800, 4194304000, 314572800, true},
+ {10485760, 262144000, 576716800, 2097152000, 419430400, false},
+ {10485760, 262144000, 576716800, 41943040000, 524288000, false},
+ {10485760, 262144000, 576716800, 419430400000, 629145600, false},
+ {10485760, 262144000, 576716800, 4194304000000, 734003200, false}
};
-#else
-// slightly modified version of Google's original
-static const int64_t kTargetFileSize = 10 * 1048576;
-
-// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
-// stop building a single file in a level->level+1 compaction.
-static const int64_t kMaxGrandParentOverlapBytes = 25 * kTargetFileSize;
-
-// Maximum number of bytes in all compacted files. We avoid expanding
-// the lower level file set of a compaction if it would make the
-// total compaction cover more than this many bytes.
-//static const int64_t kExpandedCompactionByteSizeLimit = 15 * kTargetFileSize;
-static const int64_t kExpandedCompactionByteSizeLimit = 55 * kTargetFileSize;
-
-static double MaxBytesForLevel(int level) {
- // Note: the result for level zero is not really used since we set
- // the level-0 compaction threshold based on number of files.
- double result = 200 * 1048576.0;
- if (1==level) result*=2.0;
- while (level > 1) {
- result *= 10.0;
- level--;
- }
- return result;
-}
-
-// Note: appears lower level of compaction sets size
-// for all files ... need to verify
-static uint64_t MaxFileSizeForLevel(int level) {
- return(kTargetFileSize*(level+1)*10);
-}
-#endif
-
-
-
static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
int64_t sum = 0;
for (size_t i = 0; i < files.size(); i++) {
@@ -1040,9 +1004,9 @@ void VersionSet::Finalize(Version* v) {
score*=1000000.0;
// compute penalty for write throttle if too many Level-0 files accumulating
- if ((size_t)config::kL0_CompactionTrigger <= v->files_[level].size())
+ if ((size_t)config::kL0_CompactionTrigger < v->files_[level].size())
{
- penalty+=v->files_[level].size() - config::kL0_CompactionTrigger +1;
+ penalty+=v->files_[level].size() - config::kL0_CompactionTrigger;
} // if
} else {
@@ -1052,8 +1016,8 @@ void VersionSet::Finalize(Version* v) {
// riak 1.4: new overlapped levels remove the requirement for
// aggressive penalties here, hence the retirement of "*2" and previous "*5".
- if (1<=score)
- penalty+=(static_cast<int>(score));// was *2; // was 5;
+ if (2.6<score)
+ penalty+=(static_cast<int>(score))-1;// was *2; // was *5;
}
if (score > best_score) {
@@ -1518,23 +1482,31 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
}
bool Compaction::ShouldStopBefore(const Slice& internal_key) {
- // Scan to find earliest grandparent file that contains key.
- const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
- while (grandparent_index_ < grandparents_.size() &&
- icmp->Compare(internal_key,
- grandparents_[grandparent_index_]->largest.Encode()) > 0) {
- if (seen_key_) {
- overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
+ if (!gLevelTraits[level()+1].m_OverlappedFiles)
+ {
+ // Scan to find earliest grandparent file that contains key.
+ const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
+ while (grandparent_index_ < grandparents_.size() &&
+ icmp->Compare(internal_key,
+ grandparents_[grandparent_index_]->largest.Encode()) > 0) {
+ if (seen_key_) {
+ overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
+ }
+ grandparent_index_++;
}
- grandparent_index_++;
- }
- seen_key_ = true;
+ seen_key_ = true;
- if (overlapped_bytes_ > gLevelTraits[level_].m_MaxGrandParentOverlapBytes) {
- // Too much overlap for current output; start new output
- overlapped_bytes_ = 0;
- return true;
- } else {
+ if (overlapped_bytes_ > gLevelTraits[level_].m_MaxGrandParentOverlapBytes) {
+ // Too much overlap for current output; start new output
+ overlapped_bytes_ = 0;
+ return true;
+ } else {
+ return false;
+ }
+ } // if
+ else
+ {
+ // overlapped levels do NOT split their output file
return false;
}
}
Something went wrong with that request. Please try again.