Skip to content

Commit

Permalink
Make db_wal_test slightly faster
Browse files Browse the repository at this point in the history 
Summary:
Avoid to run db_wal_test in all the DB test options, and some small changes.
Closes #1921

Differential Revision: D4622054

Pulled By: siying

fbshipit-source-id: 890fd64
  • Loading branch information
@siying @facebook-github-bot
siying authored and facebook-github-bot committed Mar 1, 2017
1 parent ba4c77b commit d5b607a
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 27 deletions.
30 changes: 30 additions & 0 deletions db/db_test_util.cc
View file
Expand Up @@ -191,6 +191,36 @@ bool DBTestBase::ChangeCompactOptions() {
}
}

// Switch between different WAL settings
bool DBTestBase::ChangeWalOptions() {
if (option_config_ == kDefault) {
option_config_ = kDBLogDir;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
options.create_if_missing = true;
TryReopen(options);
return true;
} else if (option_config_ == kDBLogDir) {
option_config_ = kWalDirAndMmapReads;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
options.create_if_missing = true;
TryReopen(options);
return true;
} else if (option_config_ == kWalDirAndMmapReads) {
option_config_ = kRecycleLogFiles;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
TryReopen(options);
return true;
} else {
return false;
}
}

// Switch between different filter policy
// Jump from kDefault to kFilter to kFullFilter
bool DBTestBase::ChangeFilterOptions() {
Expand Down
5 changes: 4 additions & 1 deletion db/db_test_util.h
View file
Expand Up @@ -671,9 +671,12 @@ class DBTestBase : public testing::Test {
// test. Return false if there are no more configurations to test.
bool ChangeOptions(int skip_mask = kNoSkip);

// Switch between different compaction styles (we have only 2 now).
// Switch between different compaction styles.
bool ChangeCompactOptions();

// Switch between different WAL-realted options.
bool ChangeWalOptions();

// Switch between different filter policy
// Jump from kDefault to kFilter to kFullFilter
bool ChangeFilterOptions();
Expand Down
53 changes: 27 additions & 26 deletions db/db_wal_test.cc
View file
Expand Up @@ -51,7 +51,7 @@ TEST_F(DBWALTest, WAL) {
// again both values should be present.
ASSERT_EQ("v3", Get(1, "foo"));
ASSERT_EQ("v3", Get(1, "bar"));
} while (ChangeCompactOptions());
} while (ChangeWalOptions());
}

TEST_F(DBWALTest, RollLog) {
Expand All @@ -68,7 +68,7 @@ TEST_F(DBWALTest, RollLog) {
for (int i = 0; i < 10; i++) {
ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
}
} while (ChangeOptions());
} while (ChangeWalOptions());
}

TEST_F(DBWALTest, SyncWALNotBlockWrite) {
Expand Down Expand Up @@ -151,7 +151,7 @@ TEST_F(DBWALTest, Recover) {
ASSERT_EQ("v4", Get(1, "foo"));
ASSERT_EQ("v2", Get(1, "bar"));
ASSERT_EQ("v5", Get(1, "baz"));
} while (ChangeOptions());
} while (ChangeWalOptions());
}

TEST_F(DBWALTest, RecoverWithTableHandle) {
Expand Down Expand Up @@ -188,23 +188,22 @@ TEST_F(DBWALTest, RecoverWithTableHandle) {
}
}
}
} while (ChangeOptions());
} while (ChangeWalOptions());
}

TEST_F(DBWALTest, IgnoreRecoveredLog) {
std::string backup_logs = dbname_ + "/backup_logs";

// delete old files in backup_logs directory
env_->CreateDirIfMissing(backup_logs);
std::vector<std::string> old_files;
env_->GetChildren(backup_logs, &old_files);
for (auto& file : old_files) {
if (file != "." && file != "..") {
env_->DeleteFile(backup_logs + "/" + file);
}
}

do {
// delete old files in backup_logs directory
env_->CreateDirIfMissing(backup_logs);
std::vector<std::string> old_files;
env_->GetChildren(backup_logs, &old_files);
for (auto& file : old_files) {
if (file != "." && file != "..") {
env_->DeleteFile(backup_logs + "/" + file);
}
}
Options options = CurrentOptions();
options.create_if_missing = true;
options.merge_operator = MergeOperators::CreateUInt64AddOperator();
Expand Down Expand Up @@ -277,7 +276,8 @@ TEST_F(DBWALTest, IgnoreRecoveredLog) {
}
Status s = TryReopen(options);
ASSERT_TRUE(!s.ok());
} while (ChangeOptions(kSkipHashCuckoo));
Destroy(options);
} while (ChangeWalOptions());
}

TEST_F(DBWALTest, RecoveryWithEmptyLog) {
Expand All @@ -290,7 +290,7 @@ TEST_F(DBWALTest, RecoveryWithEmptyLog) {
ASSERT_OK(Put(1, "foo", "v3"));
ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
ASSERT_EQ("v3", Get(1, "foo"));
} while (ChangeOptions());
} while (ChangeWalOptions());
}

#if !(defined NDEBUG) || !defined(OS_WIN)
Expand Down Expand Up @@ -429,7 +429,7 @@ TEST_F(DBWALTest, GetSortedWalFiles) {
ASSERT_OK(Put(1, "foo", "v1"));
ASSERT_OK(dbfull()->GetSortedWalFiles(log_files));
ASSERT_EQ(1, log_files.size());
} while (ChangeOptions());
} while (ChangeWalOptions());
}

TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
Expand All @@ -454,7 +454,7 @@ TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
}
// Check at least the first WAL was cleaned up during the recovery.
ASSERT_LT(earliest_log_nums[0], earliest_log_nums[1]);
} while (ChangeOptions());
} while (ChangeWalOptions());
}

TEST_F(DBWALTest, RecoverWithLargeLog) {
Expand All @@ -481,7 +481,7 @@ TEST_F(DBWALTest, RecoverWithLargeLog) {
ASSERT_EQ(std::string(10, '3'), Get(1, "small3"));
ASSERT_EQ(std::string(10, '4'), Get(1, "small4"));
ASSERT_GT(NumTableFilesAtLevel(0, 1), 1);
} while (ChangeCompactOptions());
} while (ChangeWalOptions());
}

// In https://reviews.facebook.net/D20661 we change
Expand Down Expand Up @@ -679,9 +679,9 @@ class RecoveryTestHelper {
// Starting number for the WAL file name like 00010.log
static const int kWALFileOffset = 10;
// Keys to be written per WAL file
static const int kKeysPerWALFile = 1024;
static const int kKeysPerWALFile = 133;
// Size of the value
static const int kValueSize = 10;
static const int kValueSize = 96;

// Create WAL files with values filled in
static void FillData(DBWALTest* test, const Options& options,
Expand All @@ -690,7 +690,7 @@ class RecoveryTestHelper {

*count = 0;

shared_ptr<Cache> table_cache = NewLRUCache(50000, 16);
shared_ptr<Cache> table_cache = NewLRUCache(50, 0);
EnvOptions env_options;
WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size);

Expand All @@ -717,12 +717,13 @@ class RecoveryTestHelper {
new log::Writer(std::move(file_writer), current_log_number,
db_options.recycle_log_file_num > 0));

WriteBatch batch;
for (int i = 0; i < kKeysPerWALFile; i++) {
std::string key = "key" + ToString((*count)++);
std::string value = test->DummyString(kValueSize);
assert(current_log_writer.get() != nullptr);
uint64_t seq = versions->LastSequence() + 1;
WriteBatch batch;
batch.Clear();
batch.Put(key, value);
WriteBatchInternal::SetSequence(&batch, seq);
current_log_writer->AddRecord(WriteBatchInternal::Contents(&batch));
Expand Down Expand Up @@ -773,7 +774,7 @@ class RecoveryTestHelper {
if (trunc) {
ASSERT_EQ(0, truncate(fname.c_str(), static_cast<int64_t>(size * off)));
} else {
InduceCorruption(fname, static_cast<size_t>(size * off),
InduceCorruption(fname, static_cast<size_t>(size * off + 8),
static_cast<size_t>(size * len));
}
}
Expand All @@ -792,7 +793,7 @@ class RecoveryTestHelper {
ASSERT_EQ(offset, lseek(fd, static_cast<long>(offset), SEEK_SET));

void* buf = alloca(len);
memset(buf, 'a', len);
memset(buf, 'b', len);
ASSERT_EQ(len, write(fd, buf, static_cast<unsigned int>(len)));

close(fd);
Expand All @@ -808,7 +809,7 @@ TEST_F(DBWALTest, kTolerateCorruptedTailRecords) {
const int jend = jstart + RecoveryTestHelper::kWALFilesCount;

for (auto trunc : {true, false}) { /* Corruption style */
for (int i = 0; i < 4; i++) { /* Corruption offset position */
for (int i = 0; i < 3; i++) { /* Corruption offset position */
for (int j = jstart; j < jend; j++) { /* WAL file */
// Fill data for testing
Options options = CurrentOptions();
Expand Down

0 comments on commit d5b607a

Please sign in to comment.