Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Simplify database initialization #107

Merged
merged 8 commits into from
Feb 18, 2020
Merged

Simplify database initialization #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a99e91c
update unique_ptr only if pointer is valid
Feb 12, 2020
6523a62
Create column families when opening the DB
Feb 17, 2020
ca89c2a
fixup! Create column families when opening the DB
Feb 17, 2020
b6090b6
Remove unused function
Feb 17, 2020
802580e
fix formatting;
Feb 18, 2020
6c8765a
Bump cmake-format Python package
Feb 18, 2020
91376d7
Restore documentation
Feb 18, 2020
fa9af07
Fix CMake format
Feb 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmake/hpc-coding-conventions
Open in desktop
Submodule hpc-coding-conventions updated 18 files
+0 −0 cpp/.clang-format-7
+91 −0 cpp/.clang-format-8
+92 −0 cpp/.clang-format-9
+26 −19 cpp/.cmake-format.yaml
+197 −125 cpp/CMakeLists.txt
+2 −2 cpp/NamingConventions.md
+128 −12 cpp/README.md
+15 −6 cpp/Tooling.md
+6 −0 cpp/cmake/FindClangFormat.cmake
+1 −0 cpp/cmake/FindClangTidy.cmake
+191 −7 cpp/cmake/bbp-clang-format.py
+5 −6 cpp/cmake/bbp-cmake-format.py
+60 −0 cpp/cmake/bbp-diff-filter.py
+5 −10 cpp/cmake/bbp-setup-pre-commit-config.py
+99 −96 cpp/cmake/bob.cmake
+324 −6 cpp/cmake/cpplib.py
+66 −16 dev/bump
+9 −0 dev/repositories.yaml
2 changes: 1 addition & 1 deletion dev/travis-build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-$SCRIPT_DIR/..}"

if [ "x${CPP_CHECK}" != x ] ;then
pip install --user --upgrade cmake-format==0.4.5
pip install --user --upgrade cmake-format==0.6.0
export PATH="$HOME/.local/bin:$PATH"
cmake-format --version
cd "${TRAVIS_BUILD_DIR}"
Expand Down
60 changes: 14 additions & 46 deletions src/basalt/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,6 @@ compression_if_present(const nlohmann::json& config) {
return nullptr;
}

/**
* Convert rocksdb status object to a basalt one
*/
static inline Status to_status(const rocksdb::Status& status) {
return {static_cast<Status::Code>(status.code()), status.ToString()};
}

/**
* Get real column family name from one specified in the JSON config
*/
Expand Down Expand Up @@ -295,40 +288,6 @@ static rocksdb::ColumnFamilyOptions column_families_options(
return result;
}

/**
* Prepare the rocksdb column families on the filesystem if they do not already exist.
*/
static void create_columns_families(const nlohmann::json& config,
const std::string& db_path,
rocksdb::Options& options,
std::shared_ptr<rocksdb::Cache> global_block_cache) {
std::unique_ptr<rocksdb::DB> db_ptr;
{ // open db
rocksdb::DB* db;
rocksdb::Status status = rocksdb::DB::Open(options, db_path, &db);
db_ptr.reset(db);
if (status.code() == rocksdb::Status::kInvalidArgument) {
return;
}
to_status(status).raise_on_error();
}

std::vector<std::string> column_family_names;
rocksdb::DB::ListColumnFamilies(options, db_ptr->GetName(), &column_family_names);

for (auto const& cf_config: config) {
auto const& name = column_family_name(cf_config["name"]);
if (std::find(column_family_names.begin(), column_family_names.end(), name) ==
column_family_names.end()) {
gsl::owner<rocksdb::ColumnFamilyHandle*> cf = nullptr;
auto const& cf_options = column_families_options(cf_config["config"],
global_block_cache);
to_status(db_ptr->CreateColumnFamily(cf_options, name, &cf)).raise_on_error();
delete cf;
}
}
}

/**
* Setup rocksdb statistics according to JSON config
*/
Expand All @@ -352,9 +311,18 @@ static void setup_max_open_files(const nlohmann::json& config, rocksdb::Options&
}

static void setup_create_if_missing(const nlohmann::json& config, rocksdb::Options& options) {
auto const& value = config.find("create_if_missing");
if (value != config.end()) {
options.create_if_missing = value.value().get<decltype(options.create_if_missing)>();
{
auto const& value = config.find("create_if_missing");
if (value != config.end()) {
options.create_if_missing = value.value().get<decltype(options.create_if_missing)>();
}
}
{
auto const& value = config.find("create_missing_column_families");
if (value != config.end()) {
options.create_missing_column_families =
value.value().get<decltype(options.create_missing_column_families)>();
}
}
}

Expand All @@ -375,6 +343,7 @@ static nlohmann::json default_json() {
config["statistics"] = true;
config["max_open_files"] = -1;
config["create_if_missing"] = true;
config["create_missing_column_families"] = true;
// clang-format off
config["block_cache"] = {
{"type", "lru"},
Expand Down Expand Up @@ -483,15 +452,14 @@ Config::Config()
Config::Config(std::ifstream& istr)
: config_(from_stream(istr)) {}

void Config::configure(rocksdb::Options& options, const std::string& db_path) const {
void Config::configure(rocksdb::Options& options) const {
setup_statistics(config_, options);
setup_max_open_files(config_, options);
setup_create_if_missing(config_, options);
setup_compression(config_, options);

std::shared_ptr<rocksdb::Cache> empty;
auto global_block_cache = block_cache_if_present(config_, empty);
create_columns_families(config_["column_families"], db_path, options, global_block_cache);
}

std::ostream& Config::write(std::ostream& ostr, std::streamsize indent) const {
Expand Down