Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add a file system parameter: --fs_uri to db_stress and db_bench #6878

Closed
wants to merge 3 commits into from
Closed

Add a file system parameter: --fs_uri to db_stress and db_bench #6878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
eb7dd62
Register the posix file system so it can be loaded
yhr May 18, 2020
c914f0c
Add a db_bench parameter for specifying a file system uri
yhr May 18, 2020
30b0f95
Add a db_stress parameter for specifying a file system uri
yhr May 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions db_stress_tool/db_stress_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ DECLARE_int32(compression_parallel_threads);
DECLARE_string(checksum_type);
DECLARE_string(hdfs);
DECLARE_string(env_uri);
DECLARE_string(fs_uri);
DECLARE_uint64(ops_per_thread);
DECLARE_uint64(log2_keys_per_lock);
DECLARE_uint64(max_manifest_file_size);
Expand Down
14 changes: 11 additions & 3 deletions db_stress_tool/db_stress_gflags.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,10 +601,18 @@ DEFINE_string(bottommost_compression_type, "disable",

DEFINE_string(checksum_type, "kCRC32c", "Algorithm to use to checksum blocks");

DEFINE_string(hdfs, "", "Name of hdfs environment");
DEFINE_string(hdfs, "",
"Name of hdfs environment. Mutually exclusive with"
" --env_uri and --fs_uri.");

DEFINE_string(env_uri, "",
"URI for env lookup. Mutually exclusive with --hdfs");
DEFINE_string(
env_uri, "",
"URI for env lookup. Mutually exclusive with --hdfs and --fs_uri");

DEFINE_string(fs_uri, "",
"URI for registry Filesystem lookup. Mutually exclusive"
" with --hdfs and --env_uri."
" Creates a default environment with the specified filesystem.");

DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread.");
static const bool FLAGS_ops_per_thread_dummy __attribute__((__unused__)) =
Expand Down
25 changes: 21 additions & 4 deletions db_stress_tool/db_stress_tool.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ static std::shared_ptr<ROCKSDB_NAMESPACE::DbStressEnvWrapper> env_wrapper_guard;
static std::shared_ptr<CompositeEnvWrapper> fault_env_guard;
} // namespace

static Env* GetCompositeEnv(std::shared_ptr<FileSystem> fs) {
static std::shared_ptr<Env> composite_env = NewCompositeEnv(fs);
return composite_env.get();
}

KeyGenContext key_gen_ctx;

int db_stress_tool(int argc, char** argv) {
Expand Down Expand Up @@ -63,18 +68,30 @@ int db_stress_tool(int argc, char** argv) {

Env* raw_env;

int env_opts =
!FLAGS_hdfs.empty() + !FLAGS_env_uri.empty() + !FLAGS_fs_uri.empty();
if (env_opts > 1) {
fprintf(stderr,
"Error: --hdfs, --env_uri and --fs_uri are mutually exclusive\n");
exit(1);
}

if (!FLAGS_hdfs.empty()) {
if (!FLAGS_env_uri.empty()) {
fprintf(stderr, "Cannot specify both --hdfs and --env_uri.\n");
exit(1);
}
raw_env = new ROCKSDB_NAMESPACE::HdfsEnv(FLAGS_hdfs);
} else if (!FLAGS_env_uri.empty()) {
Status s = Env::LoadEnv(FLAGS_env_uri, &raw_env, &env_guard);
if (raw_env == nullptr) {
fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
exit(1);
}
} else if (!FLAGS_fs_uri.empty()) {
std::shared_ptr<FileSystem> fs;
Status s = FileSystem::Load(FLAGS_fs_uri, &fs);
if (!s.ok()) {
fprintf(stderr, "Error: %s\n", s.ToString().c_str());
exit(1);
}
raw_env = GetCompositeEnv(fs);
} else {
raw_env = Env::Default();
}
Expand Down
12 changes: 12 additions & 0 deletions env/fs_posix.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
#include "port/port.h"
#include "rocksdb/options.h"
#include "rocksdb/slice.h"
#include "rocksdb/utilities/object_registry.h"
#include "test_util/sync_point.h"
#include "util/coding.h"
#include "util/compression_context_cache.h"
Expand Down Expand Up @@ -1046,4 +1047,15 @@ std::shared_ptr<FileSystem> FileSystem::Default() {
return default_fs_ptr;
}

#ifndef ROCKSDB_LITE
static FactoryFunc<FileSystem> posix_filesystem_reg =
ObjectLibrary::Default()->Register<FileSystem>(
"posix://.*",
[](const std::string& /* uri */, std::unique_ptr<FileSystem>* f,
std::string* /* errmsg */) {
f->reset(new PosixFileSystem());
return f->get();
});
#endif

} // namespace ROCKSDB_NAMESPACE
42 changes: 35 additions & 7 deletions tools/db_bench_tool.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -952,11 +952,17 @@ static bool ValidateTableCacheNumshardbits(const char* flagname,
DEFINE_int32(table_cache_numshardbits, 4, "");

#ifndef ROCKSDB_LITE
DEFINE_string(env_uri, "", "URI for registry Env lookup. Mutually exclusive"
" with --hdfs.");
DEFINE_string(env_uri, "",
"URI for registry Env lookup. Mutually exclusive"
" with --hdfs and --fs_uri");
DEFINE_string(fs_uri, "",
"URI for registry Filesystem lookup. Mutually exclusive"
" with --hdfs and --env_uri."
" Creates a default environment with the specified filesystem.");
#endif // ROCKSDB_LITE
DEFINE_string(hdfs, "", "Name of hdfs environment. Mutually exclusive with"
" --env_uri.");
DEFINE_string(hdfs, "",
"Name of hdfs environment. Mutually exclusive with"
" --env_uri and --fs_uri");

static std::shared_ptr<ROCKSDB_NAMESPACE::Env> env_guard;

Expand Down Expand Up @@ -986,6 +992,15 @@ DEFINE_int32(thread_status_per_interval, 0,
DEFINE_int32(perf_level, ROCKSDB_NAMESPACE::PerfLevel::kDisable,
"Level of perf collection");

#ifndef ROCKSDB_LITE
static ROCKSDB_NAMESPACE::Env* GetCompositeEnv(
std::shared_ptr<ROCKSDB_NAMESPACE::FileSystem> fs) {
static std::shared_ptr<ROCKSDB_NAMESPACE::Env> composite_env =
ROCKSDB_NAMESPACE::NewCompositeEnv(fs);
return composite_env.get();
}
#endif

static bool ValidateRateLimit(const char* flagname, double value) {
const double EPSILON = 1e-10;
if ( value < -EPSILON ) {
Expand Down Expand Up @@ -7097,15 +7112,28 @@ int db_bench_tool(int argc, char** argv) {
FLAGS_blob_db_compression_type_e =
StringToCompressionType(FLAGS_blob_db_compression_type.c_str());

if (!FLAGS_hdfs.empty() && !FLAGS_env_uri.empty()) {
fprintf(stderr, "Cannot provide both --hdfs and --env_uri.\n");
int env_opts =
!FLAGS_hdfs.empty() + !FLAGS_env_uri.empty() + !FLAGS_fs_uri.empty();
if (env_opts > 1) {
fprintf(stderr,
"Error: --hdfs, --env_uri and --fs_uri are mutually exclusive\n");
exit(1);
} else if (!FLAGS_env_uri.empty()) {
}

if (!FLAGS_env_uri.empty()) {
Status s = Env::LoadEnv(FLAGS_env_uri, &FLAGS_env, &env_guard);
if (FLAGS_env == nullptr) {
fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
exit(1);
}
} else if (!FLAGS_fs_uri.empty()) {
std::shared_ptr<FileSystem> fs;
Status s = FileSystem::Load(FLAGS_fs_uri, &fs);
if (fs == nullptr) {
fprintf(stderr, "Error: %s\n", s.ToString().c_str());
exit(1);
}
FLAGS_env = GetCompositeEnv(fs);
}
#endif // ROCKSDB_LITE
if (FLAGS_use_existing_keys && !FLAGS_use_existing_db) {
Expand Down