From fbdc3db1ee8157ba2c6e48f53ae41b222be0510d Mon Sep 17 00:00:00 2001 From: Oleksandr Solomakha Date: Tue, 27 May 2014 14:37:24 +0300 Subject: [PATCH] Migration tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit :Release Notes: Add converter into single level db engine it would start on before database and check db version for convert In case db already "single level" - would do nothing :Detailed Notes: Add converter into single level db engine it would start on before database and check db version for convert In case db already "single level" - would do nothing :Testing Performed: Tested on TV with custom build :Issues Addressed: [BHV-7829] Migration tool approach №2 Open-webOS-DCO-1.0-Signed-off-by: Oleksandr Solomakha Change-Id: I7848e396f581e4a528dd78ed6e93f699d53ee3ab Reviewed-on: https://g2g.palm.com/5815 Reviewed-by: DCO Verification Reviewed-by: Olexandr Solomakha Tested-by: Olexandr Solomakha Reviewed-by: Maksym Sditanov Reviewed-by: Oleg Yablokov --- CMakeLists.txt | 1 + files/launch/db8-maindb.conf.in | 1 + files/launch/db8-mediadb.conf.in | 1 + tool/sandwichmigrate/CMakeLists.txt | 39 +++++ tool/sandwichmigrate/Migrate.cpp | 213 ++++++++++++++++++++++++++++ 5 files changed, 255 insertions(+) create mode 100755 tool/sandwichmigrate/CMakeLists.txt create mode 100755 tool/sandwichmigrate/Migrate.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e8a5191..da98fbc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -284,5 +284,6 @@ if (WEBOS_CONFIG_BUILD_TESTS) endif() add_subdirectory(tool/dbgen) +add_subdirectory(tool/sandwichmigrate) add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure ${ctest_args}) diff --git a/files/launch/db8-maindb.conf.in b/files/launch/db8-maindb.conf.in index 402adcc..22f1b77 100644 --- a/files/launch/db8-maindb.conf.in +++ b/files/launch/db8-maindb.conf.in @@ -61,5 +61,6 @@ script fi # Use jemalloc export LD_PRELOAD=/usr/lib/libjemalloc_mt.so.0:/lib/libpthread.so.0 + @WEBOS_INSTALL_SBINDIR@/mojodb_migrate @WEBOS_INSTALL_WEBOS_SYSCONFDIR@/db8/maindb.conf exec $WEBOS_NICE @WEBOS_INSTALL_SBINDIR@/mojodb-luna -c @WEBOS_INSTALL_WEBOS_SYSCONFDIR@/db8/maindb.conf end script diff --git a/files/launch/db8-mediadb.conf.in b/files/launch/db8-mediadb.conf.in index 446dee0..e6c3bb8 100644 --- a/files/launch/db8-mediadb.conf.in +++ b/files/launch/db8-mediadb.conf.in @@ -39,5 +39,6 @@ script fi # Use jemalloc export LD_PRELOAD=/usr/lib/libjemalloc_mt.so.0:/lib/libpthread.so.0 + @WEBOS_INSTALL_SBINDIR@/mojodb_migrate @WEBOS_INSTALL_WEBOS_SYSCONFDIR@/db8/mediadb.conf exec $WEBOS_NICE @WEBOS_INSTALL_SBINDIR@/mojodb-luna -c @WEBOS_INSTALL_WEBOS_SYSCONFDIR@/db8/mediadb.conf end script diff --git a/tool/sandwichmigrate/CMakeLists.txt b/tool/sandwichmigrate/CMakeLists.txt new file mode 100755 index 0000000..78331c7 --- /dev/null +++ b/tool/sandwichmigrate/CMakeLists.txt @@ -0,0 +1,39 @@ +# @@@LICENSE +# +# Copyright (c) 2014 LG Electronics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# LICENSE@@@ + +project(mojodb_migrate CXX) + +include_directories(${CMAKE_SOURCE_DIR}/inc) + +foreach(filename ${DB_BACKEND_WRAPPER_SOURCES}) + set(DB_BACKEND_WRAPPER_SOURCES_CPP ${DB_BACKEND_WRAPPER_SOURCES_CPP} "${CMAKE_SOURCE_DIR}/${filename}") +endforeach () + +add_executable(${PROJECT_NAME} + Migrate.cpp + ${DB_BACKEND_WRAPPER_SOURCES_CPP}) + +target_link_libraries(${PROJECT_NAME} + mojocore + mojodb + ${DB_BACKEND_LIB} +) +add_definitions(-DWEBOS_PERSISTENTSTORAGEDIR="${WEBOS_INSTALL_PERSISTENTSTORAGEDIR}") + +install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME} DESTINATION ${WEBOS_INSTALL_SBINDIR}) + diff --git a/tool/sandwichmigrate/Migrate.cpp b/tool/sandwichmigrate/Migrate.cpp new file mode 100755 index 0000000..2214a43 --- /dev/null +++ b/tool/sandwichmigrate/Migrate.cpp @@ -0,0 +1,213 @@ +/* @@@LICENSE +* +* Copyright (c) 2014 LG Electronics, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* LICENSE@@@ */ + +#include "db/MojDb.h" +#include "core/MojUtil.h" +#include +#include +#include +#include + +#include +#include +#include +#include "leveldb/sandwich_db.hpp" + +MojErr doMigrate(MojObject& confObj) +{ + int idx = 0; + std::deque parts = {"indexes.ldb", "indexIds.db", "kinds.db", "objects.db", "seq.ldb", "UsageDbName"}; + + MojErr err; + int result = 0; + MojObject dbConf; + err = confObj.getRequired("db", dbConf); + MojErrCheck(err); + + MojObject dbPath; + err = dbConf.getRequired("path", dbPath); + MojErrCheck(err); + + MojString dbName; + MojString path; + err = dbPath.stringValue(path); + MojErrCheck(err); + + MojString checkFolder; + err = checkFolder.append(path); + MojErrCheck(err); + err = checkFolder.append("/indexIds.db"); + MojErrCheck(err); + MojStatT stat; + err = MojStat(checkFolder, &stat); + if (err == MojErrNotFound) + { + MojErrThrowMsg(MojErrNone, _T("Database is already converted")); + } + + MojSize pos = path.rfind('/'); + path.substring(pos, path.length()-pos, dbName); + + MojString lockFile; + lockFile.append(path); + lockFile.append("/migration.lock"); + if(!access(lockFile.data(), F_OK)) + { + MojErrThrowMsg(MojErrExists, _T("Database is locked")); + } + + //check - valid old database + for (const auto &part : parts) + { + leveldb::BottomDB db; + std::string ldbPpath(path.data()); + ldbPpath += "/"; + auto s = db.Open((ldbPpath + part).c_str()); + if (!s.ok()) + { + MojErrThrowMsg(MojErrDbIO, _T("Failed to open source database %s:%s"), (ldbPpath + part).c_str(), s.ToString().c_str()); + } + } + + MojString bkPath; + bkPath.append(WEBOS_PERSISTENTSTORAGEDIR "/db8/migration"); + bkPath.append(dbName); + + MojString execCommand; + //remove temp + execCommand.clear(); + execCommand.append("rm -r "); + execCommand.append(bkPath); + system(execCommand.data()); + + //create temporary folder + execCommand.clear(); + execCommand.append("mkdir -p "); + execCommand.append(bkPath); + result = system(execCommand.data()); + MojErrCheck(result); + + //create file identifer for identification any troubles during process + FILE* pFile = fopen(lockFile.data(), "a"); + if(!pFile) + return MojErrAccessDenied; + fclose(pFile); + + typedef leveldb::SandwichDB> BackendDbTxn; + typedef leveldb::SandwichDB BackendDb; + { + BackendDb cdb; + cdb->options.create_if_missing = true; + cdb->options.error_if_exists = true; + auto s = cdb->Open(bkPath.data()); + if (!s.ok()) + { + MojErrThrowMsg(MojErrDbIO, _T("Failed to open destination database %s:%s"), bkPath.data(), s.ToString().c_str()); + } + BackendDbTxn txnDb = cdb.ref(); + + size_t batchSize = 0; + for (const auto &part : parts) + { + leveldb::BottomDB db; + std::string ldbPpath(path.data()); + ldbPpath += "/"; + s = db.Open((ldbPpath + part).c_str()); + if (!s.ok()) + { + MojErrThrowMsg(MojErrDbIO, _T("Failed to open source database %s:%s"), (ldbPpath + part).c_str(), s.ToString().c_str()); + } + auto pdb = txnDb.use(parts[idx++]); + decltype(db)::Walker it { db }; + for (it.SeekToFirst(); it.Valid(); it.Next()) + { + pdb.Put(it.key(), it.value()); + batchSize += it.key().size(); + batchSize += it.value().size(); + if(batchSize >= 1024*1024) + { + txnDb->commit(); + batchSize = 0; + } + } + } + txnDb->commit(); + } + + //remove old-database + execCommand.clear(); + execCommand.append("rm -r "); + execCommand.append(path); + execCommand.append("/*/"); + result = system(execCommand.data()); + MojErrCheck(result); + + //copy + execCommand.clear(); + execCommand.append("cp -rf "); + execCommand.append(bkPath); + execCommand.append("/* "); + execCommand.append(path); + result = system(execCommand.data()); + MojErrCheck(result); + + //remove temp + execCommand.clear(); + execCommand.append("rm -r "); + execCommand.append(bkPath); + result = system(execCommand.data()); + MojErrCheck(result); + + return MojErrNone; +} + +int main(int argc, char**argv) +{ + if (argc != 2) { + LOG_ERROR(MSGID_DB_ERROR, 0, "Invalid arg, This program need args(config file path)"); + return -1; + } + + MojObject confObj; + + MojErr err; + + MojString confStr; + err = MojFileToString(argv[1], confStr); + if (err != MojErrNone) + { + LOG_ERROR(MSGID_DB_ERROR, 0, "Can't read configuration file"); + return 0; + } + err = confObj.fromJson(confStr); + if (err != MojErrNone) + { + LOG_ERROR(MSGID_DB_ERROR, 0, "Can't read configuration file"); + return 0; + } + + err = doMigrate(confObj); + if (err != MojErrNone) + { + LOG_ERROR(MSGID_DB_ERROR, 0, "Can't convert database"); + return 0; + } + + return 0; +} +