diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 92e07ec91..bdbb67ed8 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -173,10 +173,11 @@ aux_source_directory(models MODEL_SRC) aux_source_directory(cortex-common CORTEX_COMMON) aux_source_directory(config CONFIG_SRC) aux_source_directory(database DB_SRC) +aux_source_directory(migrations MIGR_SRC) target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) -target_sources(${TARGET_NAME} PRIVATE ${CONFIG_SRC} ${CTL_SRC} ${COMMON_SRC} ${SERVICES_SRC} ${DB_SRC}) +target_sources(${TARGET_NAME} PRIVATE ${CONFIG_SRC} ${CTL_SRC} ${COMMON_SRC} ${SERVICES_SRC} ${DB_SRC} ${MIGR_SRC}) set_target_properties(${TARGET_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR} diff --git a/engine/database/hardwares.cc b/engine/database/hardware.cc similarity index 80% rename from engine/database/hardwares.cc rename to engine/database/hardware.cc index c23aec0b7..ee68749d5 100644 --- a/engine/database/hardwares.cc +++ b/engine/database/hardware.cc @@ -1,27 +1,13 @@ -#include "hardwares.h" +#include "hardware.h" #include "database.h" #include "utils/scope_exit.h" namespace cortex::db { Hardwares::Hardwares() : db_(cortex::db::Database::GetInstance().db()) { - db_.exec( - "CREATE TABLE IF NOT EXISTS hardwares (" - "uuid TEXT PRIMARY KEY," - "type TEXT," - "hardware_id INTEGER," - "software_id INTEGER," - "activated INTEGER);"); } Hardwares::Hardwares(SQLite::Database& db) : db_(db) { - db_.exec( - "CREATE TABLE IF NOT EXISTS hardwares (" - "uuid TEXT PRIMARY KEY," - "type TEXT," - "hardware_id INTEGER," - "software_id INTEGER," - "activated INTEGER);"); } Hardwares::~Hardwares() {} @@ -35,7 +21,7 @@ Hardwares::LoadHardwareList() const { SQLite::Statement query( db_, "SELECT uuid, type, " - "hardware_id, software_id, activated FROM hardwares"); + "hardware_id, software_id, activated FROM hardware"); while (query.executeStep()) { HardwareEntry entry; @@ -57,7 +43,7 @@ cpp::result Hardwares::AddHardwareEntry( try { SQLite::Statement insert( db_, - "INSERT INTO hardwares (uuid, type, " + "INSERT INTO hardware (uuid, type, " "hardware_id, software_id, activated) VALUES (?, ?, " "?, ?, ?)"); insert.bind(1, new_entry.uuid); @@ -77,7 +63,7 @@ cpp::result Hardwares::UpdateHardwareEntry( const std::string& id, const HardwareEntry& updated_entry) { try { SQLite::Statement upd(db_, - "UPDATE hardwares " + "UPDATE hardware " "SET hardware_id = ?, software_id = ?, activated = ? " "WHERE uuid = ?"); upd.bind(1, updated_entry.hardware_id); @@ -97,7 +83,7 @@ cpp::result Hardwares::UpdateHardwareEntry( cpp::result Hardwares::DeleteHardwareEntry( const std::string& id) { try { - SQLite::Statement del(db_, "DELETE from hardwares WHERE uuid = ?"); + SQLite::Statement del(db_, "DELETE from hardware WHERE uuid = ?"); del.bind(1, id); if (del.exec() == 1) { CTL_INF("Deleted: " << id); diff --git a/engine/database/hardwares.h b/engine/database/hardware.h similarity index 100% rename from engine/database/hardwares.h rename to engine/database/hardware.h diff --git a/engine/database/models.cc b/engine/database/models.cc index a452ca1c5..b7988e949 100644 --- a/engine/database/models.cc +++ b/engine/database/models.cc @@ -9,23 +9,9 @@ namespace cortex::db { Models::Models() : db_(cortex::db::Database::GetInstance().db()) { - db_.exec( - "CREATE TABLE IF NOT EXISTS models (" - "model_id TEXT PRIMARY KEY," - "author_repo_id TEXT," - "branch_name TEXT," - "path_to_model_yaml TEXT," - "model_alias TEXT);"); } Models::Models(SQLite::Database& db) : db_(db) { - db_.exec( - "CREATE TABLE IF NOT EXISTS models (" - "model_id TEXT PRIMARY KEY," - "author_repo_id TEXT," - "branch_name TEXT," - "path_to_model_yaml TEXT," - "model_alias TEXT UNIQUE);"); } Models::~Models() {} diff --git a/engine/main.cc b/engine/main.cc index 8eab545b9..1aa024a10 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -9,6 +9,8 @@ #include "controllers/process_manager.h" #include "controllers/server.h" #include "cortex-common/cortexpythoni.h" +#include "database/database.h" +#include "migrations/migration_manager.h" #include "services/config_service.h" #include "services/file_watcher_service.h" #include "services/model_service.h" @@ -209,6 +211,15 @@ int main(int argc, char* argv[]) { // avoid printing logs to terminal is_server = true; + // check if migration is needed + if (auto res = cortex::migr::MigrationManager( + cortex::db::Database::GetInstance().db()) + .Migrate(); + res.has_error()) { + CLI_LOG("Error: " << res.error()); + return 1; + } + std::optional server_port; bool ignore_cout_log = false; for (int i = 0; i < argc; i++) { diff --git a/engine/migrations/migration_helper.cc b/engine/migrations/migration_helper.cc new file mode 100644 index 000000000..afebae5aa --- /dev/null +++ b/engine/migrations/migration_helper.cc @@ -0,0 +1,70 @@ +#include "migration_helper.h" + +namespace cortex::migr { +cpp::result MigrationHelper::BackupDatabase( + const std::string& src_db_path, const std::string& backup_db_path) { + try { + SQLite::Database src_db(src_db_path, SQLite::OPEN_READONLY); + sqlite3* backup_db; + + if (sqlite3_open(backup_db_path.c_str(), &backup_db) != SQLITE_OK) { + throw std::runtime_error("Failed to open backup database"); + } + + sqlite3_backup* backup = + sqlite3_backup_init(backup_db, "main", src_db.getHandle(), "main"); + if (!backup) { + sqlite3_close(backup_db); + throw std::runtime_error("Failed to initialize backup"); + } + + if (sqlite3_backup_step(backup, -1) != SQLITE_DONE) { + sqlite3_backup_finish(backup); + sqlite3_close(backup_db); + throw std::runtime_error("Failed to perform backup"); + } + + sqlite3_backup_finish(backup); + sqlite3_close(backup_db); + // CTL_INF("Backup completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Error during backup: " << e.what()); + return cpp::fail(e.what()); + } +} + +cpp::result MigrationHelper::RestoreDatabase( + const std::string& backup_db_path, const std::string& target_db_path) { + try { + SQLite::Database target_db(target_db_path, + SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); + sqlite3* backup_db; + + if (sqlite3_open(backup_db_path.c_str(), &backup_db) != SQLITE_OK) { + throw std::runtime_error("Failed to open backup database"); + } + + sqlite3_backup* backup = + sqlite3_backup_init(target_db.getHandle(), "main", backup_db, "main"); + if (!backup) { + sqlite3_close(backup_db); + throw std::runtime_error("Failed to initialize restore"); + } + + if (sqlite3_backup_step(backup, -1) != SQLITE_DONE) { + sqlite3_backup_finish(backup); + sqlite3_close(backup_db); + throw std::runtime_error("Failed to perform restore"); + } + + sqlite3_backup_finish(backup); + sqlite3_close(backup_db); + // CTL_INF("Restore completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Error during restore: " << e.what()); + return cpp::fail(e.what()); + } +} +} // namespace cortex::migr \ No newline at end of file diff --git a/engine/migrations/migration_helper.h b/engine/migrations/migration_helper.h new file mode 100644 index 000000000..ff0ee5075 --- /dev/null +++ b/engine/migrations/migration_helper.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include "utils/logging_utils.h" +#include "utils/result.hpp" + +namespace cortex::migr { +class MigrationHelper { + public: + cpp::result BackupDatabase( + const std::string& src_db_path, const std::string& backup_db_path); + + cpp::result RestoreDatabase( + const std::string& backup_db_path, const std::string& target_db_path); +}; +} // namespace cortex::migr diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc new file mode 100644 index 000000000..b2920722f --- /dev/null +++ b/engine/migrations/migration_manager.cc @@ -0,0 +1,224 @@ +#include "migration_manager.h" +#include +#include "assert.h" +#include "schema_version.h" +#include "utils/file_manager_utils.h" +#include "utils/scope_exit.h" + +namespace cortex::migr { + +namespace { +int GetSchemaVersion(SQLite::Database& db) { + int version = -1; // Default version if not set + + try { + SQLite::Statement query(db, "SELECT version FROM schema_version LIMIT 1;"); + + // Execute the query and get the result + if (query.executeStep()) { + version = + query.getColumn(0).getInt(); // Get the version from the first column + } + } catch (const std::exception& e) { + // CTL_WRN("SQLite error: " << e.what()); + } + + return version; +} + +constexpr const auto kCortexDb = "cortex.db"; +constexpr const auto kCortexDbBackup = "cortex_backup.db"; +} // namespace + +cpp::result MigrationManager::Migrate() { + namespace fmu = file_manager_utils; + int last_schema_version = GetSchemaVersion(db_); + int target_schema_version = SCHEMA_VERSION; + if (last_schema_version == target_schema_version) + return true; + // Back up all data before migrating + if (std::filesystem::exists(fmu::GetCortexDataPath() / kCortexDb)) { + auto src_db_path = (fmu::GetCortexDataPath() / kCortexDb).string(); + auto backup_db_path = (fmu::GetCortexDataPath() / kCortexDbBackup).string(); + if (auto res = mgr_helper_.BackupDatabase(src_db_path, backup_db_path); + res.has_error()) { + CTL_INF("Error: backup database failed!"); + return res; + } + } + + cortex::utils::ScopeExit se([]() { + auto cortex_tmp = fmu::GetCortexDataPath() / kCortexDbBackup; + if (std::filesystem::exists(cortex_tmp)) { + try { + auto n = std::filesystem::remove_all(cortex_tmp); + // CTL_INF("Deleted " << n << " files or directories"); + } catch (const std::exception& e) { + CTL_WRN(e.what()); + } + } + }); + + auto restore_db = [this]() -> cpp::result { + auto src_db_path = (fmu::GetCortexDataPath() / kCortexDb).string(); + auto backup_db_path = (fmu::GetCortexDataPath() / kCortexDbBackup).string(); + return mgr_helper_.BackupDatabase(src_db_path, backup_db_path); + }; + + // Backup folder structure + // Update logic if the folder structure changes + + // Migrate folder structure + if (last_schema_version <= target_schema_version) { + if (auto res = + UpFolderStructure(last_schema_version, target_schema_version); + res.has_error()) { + // Restore + return res; + } + } else { + if (auto res = + DownFolderStructure(last_schema_version, target_schema_version); + res.has_error()) { + // Restore + return res; + } + } + + // Update database on folder structure changes + // Update logic if the folder structure changes + + // Migrate database + if (last_schema_version < target_schema_version) { + if (auto res = UpDB(last_schema_version, target_schema_version); + res.has_error()) { + auto r = restore_db(); + return res; + } + } else { + if (auto res = DownDB(last_schema_version, target_schema_version); + res.has_error()) { + auto r = restore_db(); + return res; + } + } + return true; +} + +cpp::result MigrationManager::UpFolderStructure(int current, + int target) { + assert(current < target); + for (int v = current + 1; v <= target; v++) { + if (auto res = DoUpFolderStructure(v /*version*/); res.has_error()) { + return res; + } + } + return true; +} + +cpp::result MigrationManager::DownFolderStructure( + int current, int target) { + assert(current > target); + for (int v = current; v > target; v--) { + if (auto res = DoDownFolderStructure(v /*version*/); res.has_error()) { + return res; + } + } + return true; +} + +cpp::result MigrationManager::DoUpFolderStructure( + int version) { + switch (version) { + case 0: + return v0::MigrateFolderStructureUp(); + break; + + default: + return true; + } +} +cpp::result MigrationManager::DoDownFolderStructure( + int version) { + switch (version) { + case 0: + return v0::MigrateFolderStructureDown(); + break; + + default: + return true; + } +} + +cpp::result MigrationManager::UpDB(int current, int target) { + assert(current < target); + for (int v = current + 1; v <= target; v++) { + if (auto res = DoUpDB(v /*version*/); res.has_error()) { + return res; + } + } + // Save database + return UpdateSchemaVersion(current, target); +} +cpp::result MigrationManager::DownDB(int current, + int target) { + assert(current > target); + for (int v = current; v > target; v--) { + if (auto res = DoDownDB(v /*version*/); res.has_error()) { + return res; + } + } + // Save database + return UpdateSchemaVersion(current, target); +} + +cpp::result MigrationManager::DoUpDB(int version) { + switch (version) { + case 0: + return v0::MigrateDBUp(db_); + break; + + default: + return true; + } +} + +cpp::result MigrationManager::DoDownDB(int version) { + switch (version) { + case 0: + return v0::MigrateDBDown(db_); + break; + + default: + return true; + } +} + +cpp::result MigrationManager::UpdateSchemaVersion( + int old_version, int new_version) { + if (old_version == new_version) + return true; + try { + db_.exec("BEGIN TRANSACTION;"); + + SQLite::Statement insert(db_, + "INSERT INTO schema_version (version) VALUES (?)"); + insert.bind(1, new_version); + insert.exec(); + + if (old_version != -1) { + SQLite::Statement del(db_, + "DELETE FROM schema_version WHERE version = ?"); + del.bind(1, old_version); + del.exec(); + } + + db_.exec("COMMIT;"); + // CTL_INF("Inserted: " << version); + return true; + } catch (const std::exception& e) { + CTL_WRN(e.what()); + return cpp::fail(e.what()); + } +} +} // namespace cortex::migr \ No newline at end of file diff --git a/engine/migrations/migration_manager.h b/engine/migrations/migration_manager.h new file mode 100644 index 000000000..b05a76c26 --- /dev/null +++ b/engine/migrations/migration_manager.h @@ -0,0 +1,31 @@ +#pragma once +#include "migration_helper.h" +#include "v0/migration.h" + +namespace cortex::migr { +class MigrationManager { + public: + explicit MigrationManager(SQLite::Database& db) : db_(db) {} + cpp::result Migrate(); + + private: + cpp::result UpFolderStructure(int current, int target); + cpp::result DownFolderStructure(int current, int target); + + cpp::result DoUpFolderStructure(int version); + cpp::result DoDownFolderStructure(int version); + + cpp::result UpDB(int current, int target); + cpp::result DownDB(int current, int target); + + cpp::result DoUpDB(int version); + cpp::result DoDownDB(int version); + + cpp::result UpdateSchemaVersion(int old_version, + int new_version); + + private: + MigrationHelper mgr_helper_; + SQLite::Database& db_; +}; +} // namespace cortex::migr \ No newline at end of file diff --git a/engine/migrations/schema_version.h b/engine/migrations/schema_version.h new file mode 100644 index 000000000..7cfccf27a --- /dev/null +++ b/engine/migrations/schema_version.h @@ -0,0 +1,4 @@ +#pragma once + +//Track the current schema version +#define SCHEMA_VERSION 0 \ No newline at end of file diff --git a/engine/migrations/v0/migration.h b/engine/migrations/v0/migration.h new file mode 100644 index 000000000..9d44435c5 --- /dev/null +++ b/engine/migrations/v0/migration.h @@ -0,0 +1,88 @@ +#pragma once +#include +#include +#include +#include "utils/file_manager_utils.h" +#include "utils/logging_utils.h" +#include "utils/result.hpp" + +namespace cortex::migr::v0 { +// Data folder +namespace fmu = file_manager_utils; + +// cortexcpp +// |__ models +// | |__ cortex.so +// | |__ tinyllama +// | |__ gguf +// |__ engines +// | |__ cortex.llamacpp +// | |__ deps +// | |__ windows-amd64-avx +// |__ logs +// +inline cpp::result MigrateFolderStructureUp() { + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "models")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "models"); + } + + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "engines")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "engines"); + } + + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "logs")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "logs"); + } + + return true; +} + +inline cpp::result MigrateFolderStructureDown() { + // CTL_INF("Folder structure already up to date!"); + return true; +} + +// Database +inline cpp::result MigrateDBUp(SQLite::Database& db) { + try { + db.exec( + "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER PRIMARY " + "KEY);"); + + db.exec( + "CREATE TABLE IF NOT EXISTS models (" + "model_id TEXT PRIMARY KEY," + "author_repo_id TEXT," + "branch_name TEXT," + "path_to_model_yaml TEXT," + "model_alias TEXT);"); + + db.exec( + "CREATE TABLE IF NOT EXISTS hardware (" + "uuid TEXT PRIMARY KEY, " + "type TEXT NOT NULL, " + "hardware_id INTEGER NOT NULL, " + "software_id INTEGER NOT NULL, " + "activated INTEGER NOT NULL CHECK (activated IN (0, 1)));"); + + // CTL_INF("Database migration up completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Migration up failed: " << e.what()); + return cpp::fail(e.what()); + } +}; + +inline cpp::result MigrateDBDown(SQLite::Database& db) { + try { + db.exec("DROP TABLE IF EXISTS hardware;"); + db.exec("DROP TABLE IF EXISTS models;"); + // CTL_INF("Migration down completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Migration down failed: " << e.what()); + return cpp::fail(e.what()); + } +} + +}; // namespace cortex::migr::v0 diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc index 905b17107..02693a48d 100644 --- a/engine/services/hardware_service.cc +++ b/engine/services/hardware_service.cc @@ -7,7 +7,7 @@ #include #endif #include "cli/commands/cortex_upd_cmd.h" -#include "database/hardwares.h" +#include "database/hardware.h" #include "services/engine_service.h" #include "utils/cortex_utils.h" diff --git a/engine/test/components/test_models_db.cc b/engine/test/components/test_models_db.cc index ef54fe7e0..8c3ebbe00 100644 --- a/engine/test/components/test_models_db.cc +++ b/engine/test/components/test_models_db.cc @@ -13,7 +13,19 @@ class ModelsTestSuite : public ::testing::Test { model_list_(db_) {} void SetUp() { try { - db_.exec("DELETE FROM models"); + db_.exec( + "CREATE TABLE IF NOT EXISTS models (" + "model_id TEXT PRIMARY KEY," + "author_repo_id TEXT," + "branch_name TEXT," + "path_to_model_yaml TEXT," + "model_alias TEXT);"); + } catch (const std::exception& e) {} + } + + void TearDown() { + try { + db_.exec("DROP TABLE IF EXISTS models;"); } catch (const std::exception& e) {} }