From d72912973d000e22689e73093530434e30b0db2a Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 12 Nov 2024 15:26:11 +0700 Subject: [PATCH 1/7] feat: add schema_version --- engine/database/models.cc | 36 +++++++++++++++++++------------- engine/database/models.h | 2 ++ engine/database/schema_version.h | 4 ++++ 3 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 engine/database/schema_version.h diff --git a/engine/database/models.cc b/engine/database/models.cc index d0bee405c..bffffa560 100644 --- a/engine/database/models.cc +++ b/engine/database/models.cc @@ -3,33 +3,41 @@ #include #include #include "database.h" +#include "schema_version.h" #include "utils/result.hpp" #include "utils/scope_exit.h" namespace cortex::db { Models::Models() : db_(cortex::db::Database::GetInstance().db()) { - db_.exec( - "CREATE TABLE IF NOT EXISTS models (" - "model_id TEXT PRIMARY KEY," - "author_repo_id TEXT," - "branch_name TEXT," - "path_to_model_yaml TEXT," - "model_alias TEXT);"); + InitializeDatabase(); } Models::Models(SQLite::Database& db) : db_(db) { - db_.exec( - "CREATE TABLE IF NOT EXISTS models (" - "model_id TEXT PRIMARY KEY," - "author_repo_id TEXT," - "branch_name TEXT," - "path_to_model_yaml TEXT," - "model_alias TEXT UNIQUE);"); + InitializeDatabase(); } Models::~Models() {} +void Models::InitializeDatabase() { + switch (SCHEMA_VERSION) { + case 0: { + db_.exec( + "CREATE TABLE IF NOT EXISTS models (" + "model_id TEXT PRIMARY KEY," + "author_repo_id TEXT," + "branch_name TEXT," + "path_to_model_yaml TEXT," + "model_alias TEXT);"); + break; + } + default: { + CTL_ERR("Not supported, schema_version: " << SCHEMA_VERSION); + assert(false); + } + } +} + cpp::result, std::string> Models::LoadModelList() const { try { diff --git a/engine/database/models.h b/engine/database/models.h index 197996ab8..b7091cc82 100644 --- a/engine/database/models.h +++ b/engine/database/models.h @@ -20,6 +20,8 @@ class Models { private: SQLite::Database& db_; + void InitializeDatabase(); + bool IsUnique(const std::vector& entries, const std::string& model_id, const std::string& model_alias) const; diff --git a/engine/database/schema_version.h b/engine/database/schema_version.h new file mode 100644 index 000000000..7cfccf27a --- /dev/null +++ b/engine/database/schema_version.h @@ -0,0 +1,4 @@ +#pragma once + +//Track the current schema version +#define SCHEMA_VERSION 0 \ No newline at end of file From 8554551d8898ffc541e802b6327231246c579f35 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Thu, 14 Nov 2024 13:18:18 +0700 Subject: [PATCH 2/7] temp --- engine/database/{hardwares.cc => hardware.cc} | 24 ++----- engine/database/{hardwares.h => hardware.h} | 0 engine/database/models.cc | 22 ------- engine/database/models.h | 2 - engine/main.cc | 2 + engine/migrations/migration_helper.cc | 0 engine/migrations/migration_helper.h | 1 + engine/migrations/migration_manager.cc | 63 +++++++++++++++++++ engine/migrations/migration_manager.h | 9 +++ .../{database => migrations}/schema_version.h | 0 engine/migrations/v0/migration.h | 54 ++++++++++++++++ engine/services/hardware_service.cc | 2 +- 12 files changed, 135 insertions(+), 44 deletions(-) rename engine/database/{hardwares.cc => hardware.cc} (80%) rename engine/database/{hardwares.h => hardware.h} (100%) create mode 100644 engine/migrations/migration_helper.cc create mode 100644 engine/migrations/migration_helper.h create mode 100644 engine/migrations/migration_manager.cc create mode 100644 engine/migrations/migration_manager.h rename engine/{database => migrations}/schema_version.h (100%) create mode 100644 engine/migrations/v0/migration.h diff --git a/engine/database/hardwares.cc b/engine/database/hardware.cc similarity index 80% rename from engine/database/hardwares.cc rename to engine/database/hardware.cc index c23aec0b7..ee68749d5 100644 --- a/engine/database/hardwares.cc +++ b/engine/database/hardware.cc @@ -1,27 +1,13 @@ -#include "hardwares.h" +#include "hardware.h" #include "database.h" #include "utils/scope_exit.h" namespace cortex::db { Hardwares::Hardwares() : db_(cortex::db::Database::GetInstance().db()) { - db_.exec( - "CREATE TABLE IF NOT EXISTS hardwares (" - "uuid TEXT PRIMARY KEY," - "type TEXT," - "hardware_id INTEGER," - "software_id INTEGER," - "activated INTEGER);"); } Hardwares::Hardwares(SQLite::Database& db) : db_(db) { - db_.exec( - "CREATE TABLE IF NOT EXISTS hardwares (" - "uuid TEXT PRIMARY KEY," - "type TEXT," - "hardware_id INTEGER," - "software_id INTEGER," - "activated INTEGER);"); } Hardwares::~Hardwares() {} @@ -35,7 +21,7 @@ Hardwares::LoadHardwareList() const { SQLite::Statement query( db_, "SELECT uuid, type, " - "hardware_id, software_id, activated FROM hardwares"); + "hardware_id, software_id, activated FROM hardware"); while (query.executeStep()) { HardwareEntry entry; @@ -57,7 +43,7 @@ cpp::result Hardwares::AddHardwareEntry( try { SQLite::Statement insert( db_, - "INSERT INTO hardwares (uuid, type, " + "INSERT INTO hardware (uuid, type, " "hardware_id, software_id, activated) VALUES (?, ?, " "?, ?, ?)"); insert.bind(1, new_entry.uuid); @@ -77,7 +63,7 @@ cpp::result Hardwares::UpdateHardwareEntry( const std::string& id, const HardwareEntry& updated_entry) { try { SQLite::Statement upd(db_, - "UPDATE hardwares " + "UPDATE hardware " "SET hardware_id = ?, software_id = ?, activated = ? " "WHERE uuid = ?"); upd.bind(1, updated_entry.hardware_id); @@ -97,7 +83,7 @@ cpp::result Hardwares::UpdateHardwareEntry( cpp::result Hardwares::DeleteHardwareEntry( const std::string& id) { try { - SQLite::Statement del(db_, "DELETE from hardwares WHERE uuid = ?"); + SQLite::Statement del(db_, "DELETE from hardware WHERE uuid = ?"); del.bind(1, id); if (del.exec() == 1) { CTL_INF("Deleted: " << id); diff --git a/engine/database/hardwares.h b/engine/database/hardware.h similarity index 100% rename from engine/database/hardwares.h rename to engine/database/hardware.h diff --git a/engine/database/models.cc b/engine/database/models.cc index bffffa560..ff687baa8 100644 --- a/engine/database/models.cc +++ b/engine/database/models.cc @@ -3,41 +3,19 @@ #include #include #include "database.h" -#include "schema_version.h" #include "utils/result.hpp" #include "utils/scope_exit.h" namespace cortex::db { Models::Models() : db_(cortex::db::Database::GetInstance().db()) { - InitializeDatabase(); } Models::Models(SQLite::Database& db) : db_(db) { - InitializeDatabase(); } Models::~Models() {} -void Models::InitializeDatabase() { - switch (SCHEMA_VERSION) { - case 0: { - db_.exec( - "CREATE TABLE IF NOT EXISTS models (" - "model_id TEXT PRIMARY KEY," - "author_repo_id TEXT," - "branch_name TEXT," - "path_to_model_yaml TEXT," - "model_alias TEXT);"); - break; - } - default: { - CTL_ERR("Not supported, schema_version: " << SCHEMA_VERSION); - assert(false); - } - } -} - cpp::result, std::string> Models::LoadModelList() const { try { diff --git a/engine/database/models.h b/engine/database/models.h index b7091cc82..197996ab8 100644 --- a/engine/database/models.h +++ b/engine/database/models.h @@ -20,8 +20,6 @@ class Models { private: SQLite::Database& db_; - void InitializeDatabase(); - bool IsUnique(const std::vector& entries, const std::string& model_id, const std::string& model_alias) const; diff --git a/engine/main.cc b/engine/main.cc index e723a8fc7..f6cb42cc7 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -201,6 +201,8 @@ int main(int argc, char* argv[]) { // avoid printing logs to terminal is_server = true; + // check if migration is needed + std::optional server_port; bool ignore_cout_log = false; for (int i = 0; i < argc; i++) { diff --git a/engine/migrations/migration_helper.cc b/engine/migrations/migration_helper.cc new file mode 100644 index 000000000..e69de29bb diff --git a/engine/migrations/migration_helper.h b/engine/migrations/migration_helper.h new file mode 100644 index 000000000..7b9637ef9 --- /dev/null +++ b/engine/migrations/migration_helper.h @@ -0,0 +1 @@ +#pragma once \ No newline at end of file diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc new file mode 100644 index 000000000..8a74e0554 --- /dev/null +++ b/engine/migrations/migration_manager.cc @@ -0,0 +1,63 @@ +#include "migration_manager.h" +#include "schema_version.h" + +namespace cortex::migr { + +namespace { +cpp::result DoUp(int version) { + switch (version) { + case 0: + return v0::MigrateUp(); + break; + + default: + return true; + } +} + +cpp::result DoDown(int version) { + switch (version) { + case 0: + return v0::MigrateDown(); + break; + + default: + return true; + } +} + +int GetSchemaVersion(SQLite::Database& db) { + int version = 0; // Default version if not set + + try { + SQLite::Statement query(db, "SELECT version FROM schema_version LIMIT 1;"); + + // Execute the query and get the result + if (query.executeStep()) { + version = + query.getColumn(0).getInt(); // Get the version from the first column + } + } catch (const std::exception& e) { + std::cerr << "SQLite error: " << e.what() << std::endl; + // Handle exceptions, possibly setting a default version or taking corrective action + } + + return version; +} +} // namespace + +cpp::result MigrationManager::Up(SQLite::Database& db) { + // check schema db version + int schema_version = GetSchemaVersion(db); + int current_version = SCHEMA_VERSION; + for (int i = schema_version; i <= current_version; i++) { + if (auto res = DoUp(i /*version*/); res.has_error()) { + // Restore db and file structure + } + } + return true; +} +cpp::result MigrationManager::Down(SQLite::Database& db) { + return true; +} +} // namespace cortex::migr \ No newline at end of file diff --git a/engine/migrations/migration_manager.h b/engine/migrations/migration_manager.h new file mode 100644 index 000000000..81ff18c04 --- /dev/null +++ b/engine/migrations/migration_manager.h @@ -0,0 +1,9 @@ +#pragma once +#include "v0/migration.h" + +namespace cortex::migr { +class MigrationManager { + cpp::result Up(SQLite::Database& db); + cpp::result Down(SQLite::Database& db); +}; +} // namespace cortex::migr \ No newline at end of file diff --git a/engine/database/schema_version.h b/engine/migrations/schema_version.h similarity index 100% rename from engine/database/schema_version.h rename to engine/migrations/schema_version.h diff --git a/engine/migrations/v0/migration.h b/engine/migrations/v0/migration.h new file mode 100644 index 000000000..a35753e37 --- /dev/null +++ b/engine/migrations/v0/migration.h @@ -0,0 +1,54 @@ +#pragma once +#include +#include +#include "utils/logging_utils.h" +#include "utils/result.hpp" + +namespace cortex::migr::v0 { +// Data folder + +inline cpp::result MigrateUp() { + return true; +} + +inline cpp::result MigrateDown() { + return true; +} + +// Database +inline cpp::result MigrateUp(SQLite::Database& db) { + try { + db.exec( + "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT " + "NULL);"); + + db.exec( + "CREATE TABLE IF NOT EXISTS models (" + "model_id TEXT PRIMARY KEY," + "author_repo_id TEXT," + "branch_name TEXT," + "path_to_model_yaml TEXT," + "model_alias TEXT);"); + + db.exec( + "CREATE TABLE IF NOT EXISTS hardware (" + "uuid TEXT NOT NULL, " + "type TEXT NOT NULL, " + "hardware_id INTEGER NOT NULL, " + "software_id INTEGER NOT NULL, " + "activated INTEGER NOT NULL CHECK (activated IN (0, 1)));"); + + CTL_INF("Database migration up completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Migration up failed: " << e.what()); + return cpp::fail(e.what()); + } +}; + +inline cpp::result MigrateDown(SQLite::Database& db) { + CTL_INF("No need to drop tables for version 0"); + return true; +} + +}; // namespace cortex::migr::v0 diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc index c40133564..ba2f68e3f 100644 --- a/engine/services/hardware_service.cc +++ b/engine/services/hardware_service.cc @@ -7,7 +7,7 @@ #include #endif #include "cli/commands/cortex_upd_cmd.h" -#include "database/hardwares.h" +#include "database/hardware.h" #include "services/engine_service.h" #include "utils/cortex_utils.h" From d244ed0c09e5d079be65ea2e35b3fc1fe3dfdb04 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Fri, 15 Nov 2024 14:16:17 +0700 Subject: [PATCH 3/7] feat: data migration --- engine/CMakeLists.txt | 3 +- engine/main.cc | 9 ++ engine/migrations/migration_helper.cc | 70 ++++++++++ engine/migrations/migration_helper.h | 18 ++- engine/migrations/migration_manager.cc | 176 ++++++++++++++++++++----- engine/migrations/migration_manager.h | 25 +++- engine/migrations/v0/migration.h | 46 ++++++- 7 files changed, 307 insertions(+), 40 deletions(-) diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 92e07ec91..bdbb67ed8 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -173,10 +173,11 @@ aux_source_directory(models MODEL_SRC) aux_source_directory(cortex-common CORTEX_COMMON) aux_source_directory(config CONFIG_SRC) aux_source_directory(database DB_SRC) +aux_source_directory(migrations MIGR_SRC) target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) -target_sources(${TARGET_NAME} PRIVATE ${CONFIG_SRC} ${CTL_SRC} ${COMMON_SRC} ${SERVICES_SRC} ${DB_SRC}) +target_sources(${TARGET_NAME} PRIVATE ${CONFIG_SRC} ${CTL_SRC} ${COMMON_SRC} ${SERVICES_SRC} ${DB_SRC} ${MIGR_SRC}) set_target_properties(${TARGET_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR} diff --git a/engine/main.cc b/engine/main.cc index f6cb42cc7..e017df461 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -9,6 +9,8 @@ #include "controllers/process_manager.h" #include "controllers/server.h" #include "cortex-common/cortexpythoni.h" +#include "database/database.h" +#include "migrations/migration_manager.h" #include "services/config_service.h" #include "services/model_service.h" #include "utils/archive_utils.h" @@ -202,6 +204,13 @@ int main(int argc, char* argv[]) { is_server = true; // check if migration is needed + if (auto res = cortex::migr::MigrationManager( + cortex::db::Database::GetInstance().db()) + .Migrate(); + res.has_error()) { + CLI_LOG("Error: " << res.error()); + return 1; + } std::optional server_port; bool ignore_cout_log = false; diff --git a/engine/migrations/migration_helper.cc b/engine/migrations/migration_helper.cc index e69de29bb..a90ac6583 100644 --- a/engine/migrations/migration_helper.cc +++ b/engine/migrations/migration_helper.cc @@ -0,0 +1,70 @@ +#include "migration_helper.h" + +namespace cortex::migr { +cpp::result MigrationHelper::BackupDatabase( + const std::string& src_db_path, const std::string& backup_db_path) { + try { + SQLite::Database src_db(src_db_path, SQLite::OPEN_READONLY); + sqlite3* backup_db; + + if (sqlite3_open(backup_db_path.c_str(), &backup_db) != SQLITE_OK) { + throw std::runtime_error("Failed to open backup database"); + } + + sqlite3_backup* backup = + sqlite3_backup_init(backup_db, "main", src_db.getHandle(), "main"); + if (!backup) { + sqlite3_close(backup_db); + throw std::runtime_error("Failed to initialize backup"); + } + + if (sqlite3_backup_step(backup, -1) != SQLITE_DONE) { + sqlite3_backup_finish(backup); + sqlite3_close(backup_db); + throw std::runtime_error("Failed to perform backup"); + } + + sqlite3_backup_finish(backup); + sqlite3_close(backup_db); + CTL_INF("Backup completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Error during backup: " << e.what()); + cpp::fail(e.what()); + } +} + +cpp::result MigrationHelper::RestoreDatabase( + const std::string& backup_db_path, const std::string& target_db_path) { + try { + SQLite::Database target_db(target_db_path, + SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); + sqlite3* backup_db; + + if (sqlite3_open(backup_db_path.c_str(), &backup_db) != SQLITE_OK) { + throw std::runtime_error("Failed to open backup database"); + } + + sqlite3_backup* backup = + sqlite3_backup_init(target_db.getHandle(), "main", backup_db, "main"); + if (!backup) { + sqlite3_close(backup_db); + throw std::runtime_error("Failed to initialize restore"); + } + + if (sqlite3_backup_step(backup, -1) != SQLITE_DONE) { + sqlite3_backup_finish(backup); + sqlite3_close(backup_db); + throw std::runtime_error("Failed to perform restore"); + } + + sqlite3_backup_finish(backup); + sqlite3_close(backup_db); + CTL_INF("Restore completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Error during restore: " << e.what()); + return cpp::fail(e.what()); + } +} +} // namespace cortex::migr \ No newline at end of file diff --git a/engine/migrations/migration_helper.h b/engine/migrations/migration_helper.h index 7b9637ef9..ff0ee5075 100644 --- a/engine/migrations/migration_helper.h +++ b/engine/migrations/migration_helper.h @@ -1 +1,17 @@ -#pragma once \ No newline at end of file +#pragma once + +#include +#include +#include "utils/logging_utils.h" +#include "utils/result.hpp" + +namespace cortex::migr { +class MigrationHelper { + public: + cpp::result BackupDatabase( + const std::string& src_db_path, const std::string& backup_db_path); + + cpp::result RestoreDatabase( + const std::string& backup_db_path, const std::string& target_db_path); +}; +} // namespace cortex::migr diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc index 8a74e0554..69963b0fc 100644 --- a/engine/migrations/migration_manager.cc +++ b/engine/migrations/migration_manager.cc @@ -1,31 +1,12 @@ #include "migration_manager.h" +#include +#include "assert.h" #include "schema_version.h" +#include "utils/file_manager_utils.h" namespace cortex::migr { namespace { -cpp::result DoUp(int version) { - switch (version) { - case 0: - return v0::MigrateUp(); - break; - - default: - return true; - } -} - -cpp::result DoDown(int version) { - switch (version) { - case 0: - return v0::MigrateDown(); - break; - - default: - return true; - } -} - int GetSchemaVersion(SQLite::Database& db) { int version = 0; // Default version if not set @@ -38,7 +19,7 @@ int GetSchemaVersion(SQLite::Database& db) { query.getColumn(0).getInt(); // Get the version from the first column } } catch (const std::exception& e) { - std::cerr << "SQLite error: " << e.what() << std::endl; + CTL_WRN("SQLite error: " << e.what()); // Handle exceptions, possibly setting a default version or taking corrective action } @@ -46,18 +27,153 @@ int GetSchemaVersion(SQLite::Database& db) { } } // namespace -cpp::result MigrationManager::Up(SQLite::Database& db) { - // check schema db version - int schema_version = GetSchemaVersion(db); - int current_version = SCHEMA_VERSION; - for (int i = schema_version; i <= current_version; i++) { - if (auto res = DoUp(i /*version*/); res.has_error()) { +cpp::result MigrationManager::Migrate() { + namespace fmu = file_manager_utils; + int last_schema_version = GetSchemaVersion(db_); + int target_schema_version = SCHEMA_VERSION; + if (last_schema_version == target_schema_version) + return true; + // Back up all data before migrating + if (std::filesystem::exists(fmu::GetCortexDataPath() / "cortex.db")) { + auto src_db_path = (fmu::GetCortexDataPath() / "cortex.db").string(); + auto backup_db_path = + (fmu::GetCortexDataPath() / "cortex_backup.db").string(); + if (auto res = mgr_helper_.BackupDatabase(src_db_path, backup_db_path); + res.has_error()) { + CTL_INF("Error: backup database failed!"); + return res; + } + } + + auto restore_db = [this]() -> cpp::result { + auto src_db_path = (fmu::GetCortexDataPath() / "cortex.db").string(); + auto backup_db_path = + (fmu::GetCortexDataPath() / "cortex_backup.db").string(); + return mgr_helper_.BackupDatabase(src_db_path, backup_db_path); + }; + + // Backup folder structure + // TODO(any) update logic if the folder structure changes + + // Migrate folder structure + if (last_schema_version <= target_schema_version) { + if (auto res = + UpFolderStructure(last_schema_version, target_schema_version); + res.has_error()) { + // Restore + return res; + } + } else { + if (auto res = + DownFolderStructure(last_schema_version, target_schema_version); + res.has_error()) { + // Restore + return res; + } + } + + // Update database on folder structure changes + // TODO(any) update logic if the folder structure changes + + // Migrate database + if (last_schema_version <= target_schema_version) { + if (auto res = UpDB(last_schema_version, target_schema_version); + res.has_error()) { + restore_db(); + return res; + } + } else { + if (auto res = DownDB(last_schema_version, target_schema_version); + res.has_error()) { + restore_db(); + return res; + } + } +} + +cpp::result MigrationManager::UpFolderStructure(int current, + int target) { + assert(current <= target); + for (int i = current; i <= target; i++) { + if (auto res = DoUpFolderStructure(i /*version*/); res.has_error()) { + // Restore db and file structure + } + } + return true; +} +cpp::result MigrationManager::DownFolderStructure( + int current, int target) { + assert(current >= target); + for (int i = current; i >= target; i--) { + if (auto res = DoDownFolderStructure(i /*version*/); res.has_error()) { + // Restore db and file structure + } + } + return true; +} + +cpp::result MigrationManager::DoUpFolderStructure( + int version) { + switch (version) { + case 0: + return v0::MigrateFolderStructureUp(); + break; + + default: + return true; + } +} +cpp::result MigrationManager::DoDownFolderStructure( + int version) { + switch (version) { + case 0: + return v0::MigrateFolderStructureDown(); + break; + + default: + return true; + } +} + +cpp::result MigrationManager::UpDB(int current, int target) { + assert(current <= target); + for (int v = current; v <= target; v++) { + if (auto res = DoUpDB(v /*version*/); res.has_error()) { // Restore db and file structure } } return true; } -cpp::result MigrationManager::Down(SQLite::Database& db) { +cpp::result MigrationManager::DownDB(int current, + int target) { + assert(current >= target); + for (int v = current; v >= target; v--) { + if (auto res = DoDownDB(v /*version*/); res.has_error()) { + // Restore db and file structure + } + } return true; } + +cpp::result MigrationManager::DoUpDB(int version) { + switch (version) { + case 0: + return v0::MigrateDBUp(db_); + break; + + default: + return true; + } +} + +cpp::result MigrationManager::DoDownDB(int version) { + switch (version) { + case 0: + return v0::MigrateDBDown(db_); + break; + + default: + return true; + } +} } // namespace cortex::migr \ No newline at end of file diff --git a/engine/migrations/migration_manager.h b/engine/migrations/migration_manager.h index 81ff18c04..d47c403c3 100644 --- a/engine/migrations/migration_manager.h +++ b/engine/migrations/migration_manager.h @@ -1,9 +1,30 @@ #pragma once +#include "migration_helper.h" #include "v0/migration.h" namespace cortex::migr { class MigrationManager { - cpp::result Up(SQLite::Database& db); - cpp::result Down(SQLite::Database& db); + public: + explicit MigrationManager(SQLite::Database& db): db_(db) {} + cpp::result Migrate(); + + private: + cpp::result UpFolderStructure(int current, int target); + cpp::result DownFolderStructure(int current, int target); + + cpp::result DoUpFolderStructure(int version); + cpp::result DoDownFolderStructure(int version); + + cpp::result UpDB(int current, + int target); + cpp::result DownDB(int current, + int target); + + cpp::result DoUpDB(int version); + cpp::result DoDownDB(int version); + + private: + MigrationHelper mgr_helper_; + SQLite::Database& db_; }; } // namespace cortex::migr \ No newline at end of file diff --git a/engine/migrations/v0/migration.h b/engine/migrations/v0/migration.h index a35753e37..2c9249084 100644 --- a/engine/migrations/v0/migration.h +++ b/engine/migrations/v0/migration.h @@ -1,22 +1,49 @@ #pragma once #include +#include #include +#include "utils/file_manager_utils.h" #include "utils/logging_utils.h" #include "utils/result.hpp" namespace cortex::migr::v0 { // Data folder +namespace fmu = file_manager_utils; + +// cortexcpp +// |__ models +// | |__ cortex.so +// | |__ tinyllama +// | |__ gguf +// |__ engines +// | |__ cortex.llamacpp +// | |__ deps +// | |__ windows-amd64-avx +// |__ logs +// +inline cpp::result MigrateFolderStructureUp() { + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "models")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "models"); + } + + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "engines")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "engines"); + } + + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "logs")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "logs"); + } -inline cpp::result MigrateUp() { return true; } -inline cpp::result MigrateDown() { +inline cpp::result MigrateFolderStructureDown() { + CTL_INF("Folder structure already up to date!"); return true; } // Database -inline cpp::result MigrateUp(SQLite::Database& db) { +inline cpp::result MigrateDBUp(SQLite::Database& db) { try { db.exec( "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT " @@ -46,9 +73,16 @@ inline cpp::result MigrateUp(SQLite::Database& db) { } }; -inline cpp::result MigrateDown(SQLite::Database& db) { - CTL_INF("No need to drop tables for version 0"); - return true; +inline cpp::result MigrateDBDown(SQLite::Database& db) { + try { + db.exec("DROP TABLE IF EXISTS hardware;"); + db.exec("DROP TABLE IF EXISTS models;"); + CTL_INF("Migration down completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Migration down failed: " << e.what()); + return cpp::fail(e.what()); + } } }; // namespace cortex::migr::v0 From f95331184a21bf8af77d45c64c08d743d05a601a Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Sat, 16 Nov 2024 06:09:02 +0700 Subject: [PATCH 4/7] fix: unit tests --- engine/test/components/test_models_db.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/engine/test/components/test_models_db.cc b/engine/test/components/test_models_db.cc index ef54fe7e0..8c3ebbe00 100644 --- a/engine/test/components/test_models_db.cc +++ b/engine/test/components/test_models_db.cc @@ -13,7 +13,19 @@ class ModelsTestSuite : public ::testing::Test { model_list_(db_) {} void SetUp() { try { - db_.exec("DELETE FROM models"); + db_.exec( + "CREATE TABLE IF NOT EXISTS models (" + "model_id TEXT PRIMARY KEY," + "author_repo_id TEXT," + "branch_name TEXT," + "path_to_model_yaml TEXT," + "model_alias TEXT);"); + } catch (const std::exception& e) {} + } + + void TearDown() { + try { + db_.exec("DROP TABLE IF EXISTS models;"); } catch (const std::exception& e) {} } From 33719623ef11046c751d6e9830b564078357e40b Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Mon, 18 Nov 2024 10:54:59 +0700 Subject: [PATCH 5/7] fix: logic --- engine/migrations/migration_manager.cc | 72 +++++++++++++++++++------- engine/migrations/migration_manager.h | 1 + engine/migrations/v0/migration.h | 2 +- 3 files changed, 54 insertions(+), 21 deletions(-) diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc index 69963b0fc..1ad1dd996 100644 --- a/engine/migrations/migration_manager.cc +++ b/engine/migrations/migration_manager.cc @@ -3,12 +3,13 @@ #include "assert.h" #include "schema_version.h" #include "utils/file_manager_utils.h" +#include "utils/scope_exit.h" namespace cortex::migr { namespace { int GetSchemaVersion(SQLite::Database& db) { - int version = 0; // Default version if not set + int version = -1; // Default version if not set try { SQLite::Statement query(db, "SELECT version FROM schema_version LIMIT 1;"); @@ -25,6 +26,9 @@ int GetSchemaVersion(SQLite::Database& db) { return version; } + +constexpr const auto kCortexDb = "cortex.db"; +constexpr const auto kCortexDbBackup = "cortex_backup.db"; } // namespace cpp::result MigrationManager::Migrate() { @@ -34,10 +38,9 @@ cpp::result MigrationManager::Migrate() { if (last_schema_version == target_schema_version) return true; // Back up all data before migrating - if (std::filesystem::exists(fmu::GetCortexDataPath() / "cortex.db")) { - auto src_db_path = (fmu::GetCortexDataPath() / "cortex.db").string(); - auto backup_db_path = - (fmu::GetCortexDataPath() / "cortex_backup.db").string(); + if (std::filesystem::exists(fmu::GetCortexDataPath() / kCortexDb)) { + auto src_db_path = (fmu::GetCortexDataPath() / kCortexDb).string(); + auto backup_db_path = (fmu::GetCortexDataPath() / kCortexDbBackup).string(); if (auto res = mgr_helper_.BackupDatabase(src_db_path, backup_db_path); res.has_error()) { CTL_INF("Error: backup database failed!"); @@ -45,10 +48,21 @@ cpp::result MigrationManager::Migrate() { } } + cortex::utils::ScopeExit se([]() { + auto cortex_tmp = fmu::GetCortexDataPath() / kCortexDbBackup; + if (std::filesystem::exists(cortex_tmp)) { + try { + auto n = std::filesystem::remove_all(cortex_tmp); + CTL_INF("Deleted " << n << " files or directories"); + } catch (const std::exception& e) { + CTL_WRN(e.what()); + } + } + }); + auto restore_db = [this]() -> cpp::result { - auto src_db_path = (fmu::GetCortexDataPath() / "cortex.db").string(); - auto backup_db_path = - (fmu::GetCortexDataPath() / "cortex_backup.db").string(); + auto src_db_path = (fmu::GetCortexDataPath() / kCortexDb).string(); + auto backup_db_path = (fmu::GetCortexDataPath() / kCortexDbBackup).string(); return mgr_helper_.BackupDatabase(src_db_path, backup_db_path); }; @@ -76,25 +90,26 @@ cpp::result MigrationManager::Migrate() { // TODO(any) update logic if the folder structure changes // Migrate database - if (last_schema_version <= target_schema_version) { + if (last_schema_version < target_schema_version) { if (auto res = UpDB(last_schema_version, target_schema_version); res.has_error()) { - restore_db(); + auto r = restore_db(); return res; } } else { if (auto res = DownDB(last_schema_version, target_schema_version); res.has_error()) { - restore_db(); + auto r = restore_db(); return res; } } + return true; } cpp::result MigrationManager::UpFolderStructure(int current, int target) { assert(current <= target); - for (int i = current; i <= target; i++) { + for (int i = current; i < target; i++) { if (auto res = DoUpFolderStructure(i /*version*/); res.has_error()) { // Restore db and file structure } @@ -136,23 +151,25 @@ cpp::result MigrationManager::DoDownFolderStructure( } cpp::result MigrationManager::UpDB(int current, int target) { - assert(current <= target); - for (int v = current; v <= target; v++) { + assert(current < target); + for (int v = current + 1; v <= target; v++) { if (auto res = DoUpDB(v /*version*/); res.has_error()) { - // Restore db and file structure + return res; } } - return true; + // Save database + return UpdateSchemaVersion(target); } cpp::result MigrationManager::DownDB(int current, int target) { - assert(current >= target); - for (int v = current; v >= target; v--) { + assert(current > target); + for (int v = current; v > target; v--) { if (auto res = DoDownDB(v /*version*/); res.has_error()) { - // Restore db and file structure + return res; } } - return true; + // Save database + return UpdateSchemaVersion(target); } cpp::result MigrationManager::DoUpDB(int version) { @@ -176,4 +193,19 @@ cpp::result MigrationManager::DoDownDB(int version) { return true; } } + +cpp::result MigrationManager::UpdateSchemaVersion( + int version) { + try { + SQLite::Statement insert(db_, + "INSERT INTO schema_version (version) VALUES (?)"); + insert.bind(1, version); + insert.exec(); + CTL_INF("Inserted: " << version); + return true; + } catch (const std::exception& e) { + CTL_WRN(e.what()); + return cpp::fail(e.what()); + } +} } // namespace cortex::migr \ No newline at end of file diff --git a/engine/migrations/migration_manager.h b/engine/migrations/migration_manager.h index d47c403c3..435cd318b 100644 --- a/engine/migrations/migration_manager.h +++ b/engine/migrations/migration_manager.h @@ -23,6 +23,7 @@ class MigrationManager { cpp::result DoUpDB(int version); cpp::result DoDownDB(int version); + cpp::result UpdateSchemaVersion(int version); private: MigrationHelper mgr_helper_; SQLite::Database& db_; diff --git a/engine/migrations/v0/migration.h b/engine/migrations/v0/migration.h index 2c9249084..014a9ce1d 100644 --- a/engine/migrations/v0/migration.h +++ b/engine/migrations/v0/migration.h @@ -59,7 +59,7 @@ inline cpp::result MigrateDBUp(SQLite::Database& db) { db.exec( "CREATE TABLE IF NOT EXISTS hardware (" - "uuid TEXT NOT NULL, " + "uuid TEXT PRIMARY KEY, " "type TEXT NOT NULL, " "hardware_id INTEGER NOT NULL, " "software_id INTEGER NOT NULL, " From 03c8b69087afcae069f4f8270238594b5697f70d Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Mon, 18 Nov 2024 11:09:44 +0700 Subject: [PATCH 6/7] fix: clean --- engine/migrations/migration_manager.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc index 1ad1dd996..93b2cf0e2 100644 --- a/engine/migrations/migration_manager.cc +++ b/engine/migrations/migration_manager.cc @@ -67,7 +67,7 @@ cpp::result MigrationManager::Migrate() { }; // Backup folder structure - // TODO(any) update logic if the folder structure changes + // Update logic if the folder structure changes // Migrate folder structure if (last_schema_version <= target_schema_version) { @@ -87,7 +87,7 @@ cpp::result MigrationManager::Migrate() { } // Update database on folder structure changes - // TODO(any) update logic if the folder structure changes + // Update logic if the folder structure changes // Migrate database if (last_schema_version < target_schema_version) { @@ -108,20 +108,21 @@ cpp::result MigrationManager::Migrate() { cpp::result MigrationManager::UpFolderStructure(int current, int target) { - assert(current <= target); - for (int i = current; i < target; i++) { - if (auto res = DoUpFolderStructure(i /*version*/); res.has_error()) { - // Restore db and file structure + assert(current < target); + for (int v = current + 1; v <= target; v++) { + if (auto res = DoUpFolderStructure(v /*version*/); res.has_error()) { + return res; } } return true; } + cpp::result MigrationManager::DownFolderStructure( int current, int target) { - assert(current >= target); - for (int i = current; i >= target; i--) { - if (auto res = DoDownFolderStructure(i /*version*/); res.has_error()) { - // Restore db and file structure + assert(current > target); + for (int v = current; v > target; v--) { + if (auto res = DoDownFolderStructure(v /*version*/); res.has_error()) { + return res; } } return true; From fbd3686502328278c16b56b2e0222d4209b5dad4 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 09:54:23 +0700 Subject: [PATCH 7/7] fix: comments --- engine/migrations/migration_helper.cc | 6 +++--- engine/migrations/migration_manager.cc | 28 ++++++++++++++++++-------- engine/migrations/migration_manager.h | 12 +++++------ engine/migrations/v0/migration.h | 10 ++++----- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/engine/migrations/migration_helper.cc b/engine/migrations/migration_helper.cc index a90ac6583..afebae5aa 100644 --- a/engine/migrations/migration_helper.cc +++ b/engine/migrations/migration_helper.cc @@ -26,11 +26,11 @@ cpp::result MigrationHelper::BackupDatabase( sqlite3_backup_finish(backup); sqlite3_close(backup_db); - CTL_INF("Backup completed successfully."); + // CTL_INF("Backup completed successfully."); return true; } catch (const std::exception& e) { CTL_WRN("Error during backup: " << e.what()); - cpp::fail(e.what()); + return cpp::fail(e.what()); } } @@ -60,7 +60,7 @@ cpp::result MigrationHelper::RestoreDatabase( sqlite3_backup_finish(backup); sqlite3_close(backup_db); - CTL_INF("Restore completed successfully."); + // CTL_INF("Restore completed successfully."); return true; } catch (const std::exception& e) { CTL_WRN("Error during restore: " << e.what()); diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc index 93b2cf0e2..b2920722f 100644 --- a/engine/migrations/migration_manager.cc +++ b/engine/migrations/migration_manager.cc @@ -20,8 +20,7 @@ int GetSchemaVersion(SQLite::Database& db) { query.getColumn(0).getInt(); // Get the version from the first column } } catch (const std::exception& e) { - CTL_WRN("SQLite error: " << e.what()); - // Handle exceptions, possibly setting a default version or taking corrective action + // CTL_WRN("SQLite error: " << e.what()); } return version; @@ -53,7 +52,7 @@ cpp::result MigrationManager::Migrate() { if (std::filesystem::exists(cortex_tmp)) { try { auto n = std::filesystem::remove_all(cortex_tmp); - CTL_INF("Deleted " << n << " files or directories"); + // CTL_INF("Deleted " << n << " files or directories"); } catch (const std::exception& e) { CTL_WRN(e.what()); } @@ -159,7 +158,7 @@ cpp::result MigrationManager::UpDB(int current, int target) { } } // Save database - return UpdateSchemaVersion(target); + return UpdateSchemaVersion(current, target); } cpp::result MigrationManager::DownDB(int current, int target) { @@ -170,7 +169,7 @@ cpp::result MigrationManager::DownDB(int current, } } // Save database - return UpdateSchemaVersion(target); + return UpdateSchemaVersion(current, target); } cpp::result MigrationManager::DoUpDB(int version) { @@ -196,13 +195,26 @@ cpp::result MigrationManager::DoDownDB(int version) { } cpp::result MigrationManager::UpdateSchemaVersion( - int version) { + int old_version, int new_version) { + if (old_version == new_version) + return true; try { + db_.exec("BEGIN TRANSACTION;"); + SQLite::Statement insert(db_, "INSERT INTO schema_version (version) VALUES (?)"); - insert.bind(1, version); + insert.bind(1, new_version); insert.exec(); - CTL_INF("Inserted: " << version); + + if (old_version != -1) { + SQLite::Statement del(db_, + "DELETE FROM schema_version WHERE version = ?"); + del.bind(1, old_version); + del.exec(); + } + + db_.exec("COMMIT;"); + // CTL_INF("Inserted: " << version); return true; } catch (const std::exception& e) { CTL_WRN(e.what()); diff --git a/engine/migrations/migration_manager.h b/engine/migrations/migration_manager.h index 435cd318b..b05a76c26 100644 --- a/engine/migrations/migration_manager.h +++ b/engine/migrations/migration_manager.h @@ -5,7 +5,7 @@ namespace cortex::migr { class MigrationManager { public: - explicit MigrationManager(SQLite::Database& db): db_(db) {} + explicit MigrationManager(SQLite::Database& db) : db_(db) {} cpp::result Migrate(); private: @@ -15,15 +15,15 @@ class MigrationManager { cpp::result DoUpFolderStructure(int version); cpp::result DoDownFolderStructure(int version); - cpp::result UpDB(int current, - int target); - cpp::result DownDB(int current, - int target); + cpp::result UpDB(int current, int target); + cpp::result DownDB(int current, int target); cpp::result DoUpDB(int version); cpp::result DoDownDB(int version); - cpp::result UpdateSchemaVersion(int version); + cpp::result UpdateSchemaVersion(int old_version, + int new_version); + private: MigrationHelper mgr_helper_; SQLite::Database& db_; diff --git a/engine/migrations/v0/migration.h b/engine/migrations/v0/migration.h index 014a9ce1d..9d44435c5 100644 --- a/engine/migrations/v0/migration.h +++ b/engine/migrations/v0/migration.h @@ -38,7 +38,7 @@ inline cpp::result MigrateFolderStructureUp() { } inline cpp::result MigrateFolderStructureDown() { - CTL_INF("Folder structure already up to date!"); + // CTL_INF("Folder structure already up to date!"); return true; } @@ -46,8 +46,8 @@ inline cpp::result MigrateFolderStructureDown() { inline cpp::result MigrateDBUp(SQLite::Database& db) { try { db.exec( - "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT " - "NULL);"); + "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER PRIMARY " + "KEY);"); db.exec( "CREATE TABLE IF NOT EXISTS models (" @@ -65,7 +65,7 @@ inline cpp::result MigrateDBUp(SQLite::Database& db) { "software_id INTEGER NOT NULL, " "activated INTEGER NOT NULL CHECK (activated IN (0, 1)));"); - CTL_INF("Database migration up completed successfully."); + // CTL_INF("Database migration up completed successfully."); return true; } catch (const std::exception& e) { CTL_WRN("Migration up failed: " << e.what()); @@ -77,7 +77,7 @@ inline cpp::result MigrateDBDown(SQLite::Database& db) { try { db.exec("DROP TABLE IF EXISTS hardware;"); db.exec("DROP TABLE IF EXISTS models;"); - CTL_INF("Migration down completed successfully."); + // CTL_INF("Migration down completed successfully."); return true; } catch (const std::exception& e) { CTL_WRN("Migration down failed: " << e.what());