Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Windows cumulative patch #3552

Closed
wants to merge 15 commits into from
Closed

Windows cumulative patch #3552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
f5ca24a
Close info_log logger so DestroyDB can cleanup the directory and let
yuslepukhin Feb 13, 2018
68eae93
Adjust windows logger for the new interface pattern.
yuslepukhin Feb 28, 2018
01d1830
Replace std::thread reference and fix the %z formatting.
yuslepukhin Feb 28, 2018
02d9480
Bring scope status var fixes.
yuslepukhin Feb 17, 2018
aeac697
Make use of GetFileInformationByHandleEx() so we make use of the unique
yuslepukhin Feb 21, 2018
d5bdef6
Replace C-runtime in env implementation.
yuslepukhin Feb 22, 2018
5309831
Implement AreFilesTheSame
yuslepukhin Mar 1, 2018
8e66b3e
Align unbuffered read offsets to sector size.
yuslepukhin Feb 16, 2018
0d13a75
Fix DBTest.SoftLimit. The test is based on a premise that
yuslepukhin Mar 1, 2018
0a564b2
Address windows release unused var warning.
yuslepukhin Mar 2, 2018
8f4a373
Address #endif extra chars error
yuslepukhin Mar 2, 2018
93ae4cf
Adjust for MINGW builds.
yuslepukhin Mar 2, 2018
df06f9f
Address DBTest2.ReadAmpBitmapLiveInCacheAfterDBClose failure
yuslepukhin Mar 2, 2018
7469117
Fix WriteStallListenermember initialization.
yuslepukhin Mar 5, 2018
c0c4b8e
Reset expected_set_ in a signalling thread unless there is a timeout
yuslepukhin Mar 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions build_tools/run_ci_db_test.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ $InvokeTestAsync = {
# Test limiting factor here
[int]$count = 0
# Overall status
[bool]$success = $true;
[bool]$script:success = $true;

function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
{
Expand Down Expand Up @@ -425,7 +425,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
$log_content = @(Get-Content $log)

if($completed.State -ne "Completed") {
$success = $false
$script:success = $false
Write-Warning $message
$log_content | Write-Warning
} else {
Expand All @@ -449,7 +449,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
}

if(!$pass_found) {
$success = $false;
$script:success = $false;
Write-Warning $message
$log_content | Write-Warning
} else {
Expand All @@ -473,7 +473,7 @@ New-TimeSpan -Start $StartDate -End $EndDate |
}


if(!$success) {
if(!$script:success) {
# This does not succeed killing off jobs quick
# So we simply exit
# Remove-Job -Job $jobs -Force
Expand Down
5 changes: 4 additions & 1 deletion db/db_impl.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2389,10 +2389,13 @@ Snapshot::~Snapshot() {
}

Status DestroyDB(const std::string& dbname, const Options& options) {
const ImmutableDBOptions soptions(SanitizeOptions(dbname, options));
ImmutableDBOptions soptions(SanitizeOptions(dbname, options));
Env* env = soptions.env;
std::vector<std::string> filenames;

// Reset the logger because it holds a handle to the
// log file and prevents cleanup and directory removal
soptions.info_log.reset();
// Ignore error in case directory does not exist
env->GetChildren(dbname, &filenames);

Expand Down
40 changes: 32 additions & 8 deletions db/db_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5153,7 +5153,7 @@ TEST_F(DBTest, AutomaticConflictsWithManualCompaction) {
}
ASSERT_OK(Flush());
}
std::thread manual_compaction_thread([this]() {
port::Thread manual_compaction_thread([this]() {
CompactRangeOptions croptions;
croptions.exclusive_manual_compaction = true;
ASSERT_OK(db_->CompactRange(croptions, nullptr, nullptr));
Expand Down Expand Up @@ -5393,18 +5393,42 @@ TEST_F(DBTest, HardLimit) {
#ifndef ROCKSDB_LITE
class WriteStallListener : public EventListener {
public:
WriteStallListener() : condition_(WriteStallCondition::kNormal) {}
WriteStallListener() : cond_(&mutex_),
condition_(WriteStallCondition::kNormal),
expected_(WriteStallCondition::kNormal),
expected_set_(false)
{}
void OnStallConditionsChanged(const WriteStallInfo& info) override {
MutexLock l(&mutex_);
condition_ = info.condition.cur;
if (expected_set_ &&
condition_ == expected_) {
cond_.Signal();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yuslepukhin ASAN complains this:

Failed to find test summary. This test binary may have crashed mid-run. Full output follows:
Note: Google Test filter = DBTest.SoftLimit
[==========] Running 1 test from 1 test case.
[----------] Global test environment set-up.
[----------] 1 test from DBTest
[ RUN      ] DBTest.SoftLimit
internal_repo_rocksdb/repo/db/db_test.cc:5391:9: runtime error: load of value 190, which is not a valid value for type 'bool'
    #0 0x681976 in rocksdb::WriteStallListener::OnStallConditionsChanged(rocksdb::WriteStallInfo const&) internal_repo_rocksdb/repo/db/db_test.cc:5391
    #1 0x7f7cf9d91494 in rocksdb::SuperVersionContext::Clean() internal_repo_rocksdb/repo/db/job_context.h:63
    #2 0x7f7cf9d826e2 in rocksdb::JobContext::Clean() internal_repo_rocksdb/repo/db/job_context.h:162
    #3 0x7f7cf9f33d83 in rocksdb::DBImpl::BackgroundCallFlush() internal_repo_rocksdb/repo/db/db_impl_compaction_flush.cc:1404
    #4 0x7f7cf9f30fe5 in rocksdb::DBImpl::BGWorkFlush(void*) internal_repo_rocksdb/repo/db/db_impl_compaction_flush.cc:1250
    #5 0x7f7cfa6af0f1 in rocksdb::ThreadPoolImpl::Schedule(void (*)(void*), void*, void*, void (*)(void*))::$_1::operator()() const internal_repo_rocksdb/repo/util/threadpool_imp.cc:441
    #6 0x7f7cfa6aeefc in std::_Function_handler<void (), rocksdb::ThreadPoolImpl::Schedule(void (*)(void*), void*, void*, void (*)(void*))::$_1>::_M_invoke(std::_Any_data const&) third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/trunk/functional:1871
    #7 0x7f7cfa18325e in std::function<void ()>::operator()() const third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/trunk/functional:2267
    #8 0x7f7cfa6ab557 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) internal_repo_rocksdb/repo/util/threadpool_imp.cc:240
    #9 0x7f7cfa6ab839 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) internal_repo_rocksdb/repo/util/threadpool_imp.cc:278
    #10 0x7f7cfa6b560e in void* std::_Bind_simple<void* (*(rocksdb::BGThreadMetadata*))(void*)>::_M_invoke<0ul>(std::_Index_tuple<0ul>) third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/trunk/functional:1530
    #11 0x7f7cfa6b5574 in std::_Bind_simple<void* (*(rocksdb::BGThreadMetadata*))(void*)>::operator()() third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/trunk/functional:1520
    #12 0x7f7cfa6b52a8 in std::thread::_Impl<std::_Bind_simple<void* (*(rocksdb::BGThreadMetadata*))(void*)> >::_M_run() third-party-buck/gcc-5-glibc-2.23/build/libgcc/include/c++/trunk/thread:115
    #13 0x7f7cf6e56170 in execute_native_thread_routine /home/engshare/third-party2/libgcc/5.x/src/gcc-5/x86_64-facebook-linux/libstdc++-v3/src/c++11/../../../.././libstdc++-v3/src/c++11/thread.cc:84
    #14 0x7f7cf4cab7a8 in start_thread /home/engshare/third-party2/glibc/2.23/src/glibc-2.23/nptl/pthread_create.c:333
    #15 0x7f7cf44d7a7c in __clone /home/engshare/third-party2/glibc/2.23/src/glibc-2.23/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:109

I assume it's because expected_set_ has never been initialized?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@siying My bad.

expected_set_ = false;
}
}
bool CheckCondition(WriteStallCondition expected) {
MutexLock l(&mutex_);
return expected == condition_;
if (expected != condition_) {
expected_ = expected;
expected_set_ = true;
while (expected != condition_) {
// We bail out on timeout 500 milliseconds
const uint64_t timeout_us = 500000;
if (cond_.TimedWait(timeout_us)) {
expected_set_ = false;
return false;
}
}
}
return true;
}
private:
port::Mutex mutex_;
port::Mutex mutex_;
port::CondVar cond_;
WriteStallCondition condition_;
WriteStallCondition expected_;
bool expected_set_;
};

TEST_F(DBTest, SoftLimit) {
Expand Down Expand Up @@ -5743,7 +5767,7 @@ TEST_F(DBTest, ThreadLocalPtrDeadlock) {
return flushes_done.load() > 10;
};

std::thread flushing_thread([&] {
port::Thread flushing_thread([&] {
for (int i = 0; !done(); ++i) {
ASSERT_OK(db_->Put(WriteOptions(), Slice("hi"),
Slice(std::to_string(i).c_str())));
Expand All @@ -5753,12 +5777,12 @@ TEST_F(DBTest, ThreadLocalPtrDeadlock) {
}
});

std::vector<std::thread> thread_spawning_threads(10);
std::vector<port::Thread> thread_spawning_threads(10);
for (auto& t: thread_spawning_threads) {
t = std::thread([&] {
t = port::Thread([&] {
while (!done()) {
{
std::thread tmp_thread([&] {
port::Thread tmp_thread([&] {
auto it = db_->NewIterator(ReadOptions());
delete it;
});
Expand Down
14 changes: 13 additions & 1 deletion db/db_test2.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1822,14 +1822,26 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
{
const int kIdBufLen = 100;
char id_buf[kIdBufLen];
#ifndef OS_WIN
// You can't open a directory on windows using random access file
std::unique_ptr<RandomAccessFile> file;
env_->NewRandomAccessFile(dbname_, &file, EnvOptions());
ASSERT_OK(env_->NewRandomAccessFile(dbname_, &file, EnvOptions()));
if (file->GetUniqueId(id_buf, kIdBufLen) == 0) {
// fs holding db directory doesn't support getting a unique file id,
// this means that running this test will fail because lru_cache will load
// the blocks again regardless of them being already in the cache
return;
}
#else
std::unique_ptr<Directory> dir;
ASSERT_OK(env_->NewDirectory(dbname_, &dir));
if (dir->GetUniqueId(id_buf, kIdBufLen) == 0) {
// fs holding db directory doesn't support getting a unique file id,
// this means that running this test will fail because lru_cache will load
// the blocks again regardless of them being already in the cache
return;
}
#endif
}
uint32_t bytes_per_bit[2] = {1, 16};
for (size_t k = 0; k < 2; k++) {
Expand Down
31 changes: 31 additions & 0 deletions env/env_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,37 @@ TEST_F(EnvPosixTest, RunImmediately) {
}
}

#ifdef OS_WIN
TEST_F(EnvPosixTest, AreFilesSame) {
{
bool tmp;
if (env_->AreFilesSame("", "", &tmp).IsNotSupported()) {
fprintf(stderr,
"skipping EnvBasicTestWithParam.AreFilesSame due to "
"unsupported Env::AreFilesSame\n");
return;
}
}

const EnvOptions soptions;
auto* env = Env::Default();
std::string same_file_name = test::TmpDir(env) + "/same_file";
std::string same_file_link_name = same_file_name + "_link";

std::unique_ptr<WritableFile> same_file;
ASSERT_OK(env->NewWritableFile(same_file_name,
&same_file, soptions));
same_file->Append("random_data");
ASSERT_OK(same_file->Flush());
same_file.reset();

ASSERT_OK(env->LinkFile(same_file_name, same_file_link_name));
bool result = false;
ASSERT_OK(env->AreFilesSame(same_file_name, same_file_link_name, &result));
ASSERT_TRUE(result);
}
#endif

TEST_P(EnvPosixTestWithParam, UnSchedule) {
std::atomic<bool> called(false);
env_->SetBackgroundThreads(1, Env::LOW);
Expand Down
4 changes: 4 additions & 0 deletions include/rocksdb/env.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -802,6 +802,10 @@ class Directory {
virtual ~Directory() {}
// Fsync directory. Can be called concurrently from multiple threads.
virtual Status Fsync() = 0;

virtual size_t GetUniqueId(char* id, size_t max_size) const {
return 0;
}
};

enum InfoLogLevel : unsigned char {
Expand Down
Loading