From 052647191c34126613a810306478220cf4e66815 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Wed, 4 Dec 2024 10:56:35 +0700
Subject: [PATCH 01/21] feat: prioritize GPUs

---
 engine/controllers/hardware.cc      |  2 +-
 engine/services/hardware_service.cc | 31 +++++++++++++++++++++++------
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/engine/controllers/hardware.cc b/engine/controllers/hardware.cc
index 4f5cc2879..3583995cc 100644
--- a/engine/controllers/hardware.cc
+++ b/engine/controllers/hardware.cc
@@ -40,7 +40,7 @@ void Hardware::Activate(
       ahc.gpus.push_back(g.asInt());
     }
   }
-  std::sort(ahc.gpus.begin(), ahc.gpus.end());
+  
   if (!hw_svc_->IsValidConfig(ahc)) {
     Json::Value ret;
     ret["message"] = "Invalid GPU index provided.";
diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc
index 681ca7578..5574472bd 100644
--- a/engine/services/hardware_service.cc
+++ b/engine/services/hardware_service.cc
@@ -191,13 +191,30 @@ bool HardwareService::Restart(const std::string& host, int port) {
   return true;
 }
 
+// GPU identifiers are given as integer indices or as UUID strings. GPU UUID strings
+// should follow the same format as given by nvidia-smi, such as GPU-8932f937-d72c-4106-c12f-20bd9faed9f6.
+// However, for convenience, abbreviated forms are allowed; simply specify enough digits
+// from the beginning of the GPU UUID to uniquely identify that GPU in the target system.
+// For example, CUDA_VISIBLE_DEVICES=GPU-8932f937 may be a valid way to refer to the above GPU UUID,
+// assuming no other GPU in the system shares this prefix. Only the devices whose index
+// is present in the sequence are visible to CUDA applications and they are enumerated
+// in the order of the sequence. If one of the indices is invalid, only the devices whose
+// index precedes the invalid index are visible to CUDA applications. For example, setting
+// CUDA_VISIBLE_DEVICES to 2,1 causes device 0 to be invisible and device 2 to be enumerated
+// before device 1. Setting CUDA_VISIBLE_DEVICES to 0,2,-1,1 causes devices 0 and 2 to be
+// visible and device 1 to be invisible. MIG format starts with MIG keyword and GPU UUID
+// should follow the same format as given by nvidia-smi.
+// For example, MIG-GPU-8932f937-d72c-4106-c12f-20bd9faed9f6/1/2.
+// Only single MIG instance enumeration is supported.
 bool HardwareService::SetActivateHardwareConfig(
     const cortex::hw::ActivateHardwareConfig& ahc) {
   // Note: need to map software_id and hardware_id
   // Update to db
   cortex::db::Hardwares hw_db;
-  auto activate = [&ahc](int software_id) {
-    return std::count(ahc.gpus.begin(), ahc.gpus.end(), software_id) > 0;
+  // copy all gpu information to new vector
+  auto ahc_gpus = ahc.gpus;
+  auto activate = [&ahc_gpus](int software_id) {
+    return std::count(ahc_gpus.begin(), ahc_gpus.end(), software_id) > 0;
   };
   auto res = hw_db.LoadHardwareList();
   if (res.has_value()) {
@@ -210,11 +227,12 @@ bool HardwareService::SetActivateHardwareConfig(
       }
     }
     std::sort(activated_ids.begin(), activated_ids.end());
-    if (ahc.gpus.size() != activated_ids.size()) {
+    std::sort(ahc_gpus.begin(), ahc_gpus.end());
+    if (ahc_gpus.size() != activated_ids.size()) {
       need_update = true;
     } else {
-      for (size_t i = 0; i < ahc.gpus.size(); i++) {
-        if (ahc.gpus[i] != activated_ids[i])
+      for (size_t i = 0; i < ahc_gpus.size(); i++) {
+        if (ahc_gpus[i] != activated_ids[i])
           need_update = true;
       }
     }
@@ -291,7 +309,8 @@ void HardwareService::UpdateHardwareInfos() {
   }
 
 #if defined(_WIN32) || defined(_WIN64) || defined(__linux__)
-  if (!gpus.empty()) {
+  bool has_deactivated_gpu = a.value().size() != activated_gpu_af.size();
+  if (!gpus.empty() && has_deactivated_gpu) {
     const char* value = std::getenv("CUDA_VISIBLE_DEVICES");
     if (value) {
       LOG_INFO << "CUDA_VISIBLE_DEVICES: " << value;

From df6d174df7e63d614e1db6ff05597acf9bb6dd96 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Wed, 4 Dec 2024 13:50:36 +0700
Subject: [PATCH 02/21] fix: migrate db

---
 engine/migrations/db_helper.h          |  31 ++++
 engine/migrations/migration_manager.cc |  13 ++
 engine/migrations/schema_version.h     |   2 +-
 engine/migrations/v2/migration.h       | 204 +++++++++++++++++++++++++
 4 files changed, 249 insertions(+), 1 deletion(-)
 create mode 100644 engine/migrations/db_helper.h
 create mode 100644 engine/migrations/v2/migration.h

diff --git a/engine/migrations/db_helper.h b/engine/migrations/db_helper.h
new file mode 100644
index 000000000..31254eb0f
--- /dev/null
+++ b/engine/migrations/db_helper.h
@@ -0,0 +1,31 @@
+#pragma once
+#include <SQLiteCpp/SQLiteCpp.h>
+
+namespace cortex::mgr {
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+inline bool ColumnExists(SQLite::Database& db, const std::string& table_name,
+                         const std::string& column_name) {
+  try {
+    SQLite::Statement query(
+        db, "SELECT " + column_name + " FROM " + table_name + " LIMIT 0");
+    return true;
+  } catch (std::exception&) {
+    return false;
+  }
+}
+
+inline void AddColumnIfNotExists(SQLite::Database& db,
+                                 const std::string& table_name,
+                                 const std::string& column_name,
+                                 const std::string& column_type) {
+  if (!ColumnExists(db, table_name, column_name)) {
+    std::string sql = "ALTER TABLE " + table_name + " ADD COLUMN " +
+                      column_name + " " + column_type;
+    db.exec(sql);
+  }
+}
+}  // namespace cortex::mgr
\ No newline at end of file
diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc
index 2c2b6ddfd..da5df3368 100644
--- a/engine/migrations/migration_manager.cc
+++ b/engine/migrations/migration_manager.cc
@@ -5,6 +5,7 @@
 #include "utils/file_manager_utils.h"
 #include "utils/scope_exit.h"
 #include "utils/widechar_conv.h"
+#include "v2/migration.h"
 
 namespace cortex::migr {
 
@@ -140,6 +141,9 @@ cpp::result<bool, std::string> MigrationManager::DoUpFolderStructure(
     case 0:
       return v0::MigrateFolderStructureUp();
       break;
+    case 2:
+      return v2::MigrateFolderStructureUp();
+      break;
 
     default:
       return true;
@@ -151,6 +155,9 @@ cpp::result<bool, std::string> MigrationManager::DoDownFolderStructure(
     case 0:
       return v0::MigrateFolderStructureDown();
       break;
+    case 2:
+      return v2::MigrateFolderStructureDown();
+      break;
 
     default:
       return true;
@@ -184,6 +191,9 @@ cpp::result<bool, std::string> MigrationManager::DoUpDB(int version) {
     case 0:
       return v0::MigrateDBUp(db_);
       break;
+    case 2:
+      return v2::MigrateDBUp(db_);
+      break;
 
     default:
       return true;
@@ -195,6 +205,9 @@ cpp::result<bool, std::string> MigrationManager::DoDownDB(int version) {
     case 0:
       return v0::MigrateDBDown(db_);
       break;
+    case 2:
+      return v2::MigrateDBDown(db_);
+      break;
 
     default:
       return true;
diff --git a/engine/migrations/schema_version.h b/engine/migrations/schema_version.h
index 7cfccf27a..6f07aa868 100644
--- a/engine/migrations/schema_version.h
+++ b/engine/migrations/schema_version.h
@@ -1,4 +1,4 @@
 #pragma once
 
 //Track the current schema version
-#define SCHEMA_VERSION 0
\ No newline at end of file
+#define SCHEMA_VERSION 2
\ No newline at end of file
diff --git a/engine/migrations/v2/migration.h b/engine/migrations/v2/migration.h
new file mode 100644
index 000000000..11bd0298d
--- /dev/null
+++ b/engine/migrations/v2/migration.h
@@ -0,0 +1,204 @@
+#pragma once
+#include <SQLiteCpp/Database.h>
+#include <filesystem>
+#include <string>
+#include "migrations/db_helper.h"
+#include "utils/file_manager_utils.h"
+#include "utils/logging_utils.h"
+#include "utils/result.hpp"
+
+namespace cortex::migr::v2 {
+// Data folder
+namespace fmu = file_manager_utils;
+
+// cortexcpp
+//   |__ models
+//   |     |__ cortex.so
+//   |          |__ tinyllama
+//   |                |__ gguf
+//   |__ engines
+//   |     |__ cortex.llamacpp
+//   |           |__ deps
+//   |           |__ windows-amd64-avx
+//   |__ logs
+//
+inline cpp::result<bool, std::string> MigrateFolderStructureUp() {
+  if (!std::filesystem::exists(fmu::GetCortexDataPath() / "models")) {
+    std::filesystem::create_directory(fmu::GetCortexDataPath() / "models");
+  }
+
+  if (!std::filesystem::exists(fmu::GetCortexDataPath() / "engines")) {
+    std::filesystem::create_directory(fmu::GetCortexDataPath() / "engines");
+  }
+
+  if (!std::filesystem::exists(fmu::GetCortexDataPath() / "logs")) {
+    std::filesystem::create_directory(fmu::GetCortexDataPath() / "logs");
+  }
+
+  return true;
+}
+
+inline cpp::result<bool, std::string> MigrateFolderStructureDown() {
+  // CTL_INF("Folder structure already up to date!");
+  return true;
+}
+
+// Database
+inline cpp::result<bool, std::string> MigrateDBUp(SQLite::Database& db) {
+  try {
+    db.exec(
+        "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER PRIMARY "
+        "KEY);");
+
+    // models
+    {
+      // Check if the table exists
+      SQLite::Statement query(db,
+                              "SELECT name FROM sqlite_master WHERE "
+                              "type='table' AND name='models'");
+      auto table_exists = query.executeStep();
+
+      if (table_exists) {
+        // Alter existing table
+        cortex::mgr::AddColumnIfNotExists(db, "models", "model_format", "TEXT");
+        cortex::mgr::AddColumnIfNotExists(db, "models", "model_source", "TEXT");
+        cortex::mgr::AddColumnIfNotExists(db, "models", "status", "TEXT");
+        cortex::mgr::AddColumnIfNotExists(db, "models", "engine", "TEXT");
+      } else {
+        // Create new table
+        db.exec(
+            "CREATE TABLE models ("
+            "model_id TEXT PRIMARY KEY,"
+            "author_repo_id TEXT,"
+            "branch_name TEXT,"
+            "path_to_model_yaml TEXT,"
+            "model_alias TEXT,"
+            "model_format TEXT,"
+            "model_source TEXT,"
+            "status TEXT,"
+            "engine TEXT"
+            ")");
+      }
+    }
+
+    // Check if the table exists
+    SQLite::Statement hw_query(db,
+                               "SELECT name FROM sqlite_master WHERE "
+                               "type='table' AND name='hardware'");
+    auto hw_table_exists = hw_query.executeStep();
+
+    if (hw_table_exists) {
+      // Alter existing table
+      cortex::mgr::AddColumnIfNotExists(db, "hardware", "priority", "INTEGER");
+    } else {
+      db.exec(
+          "CREATE TABLE IF NOT EXISTS hardware ("
+          "uuid TEXT PRIMARY KEY, "
+          "type TEXT NOT NULL, "
+          "hardware_id INTEGER NOT NULL, "
+          "software_id INTEGER NOT NULL, "
+          "activated INTEGER NOT NULL CHECK (activated IN (0, 1)), "
+          "priority INTEGER); ");
+    }
+
+    // engines
+    db.exec(
+        "CREATE TABLE IF NOT EXISTS engines ("
+        "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        "engine_name TEXT,"
+        "type TEXT,"
+        "api_key TEXT,"
+        "url TEXT,"
+        "version TEXT,"
+        "variant TEXT,"
+        "status TEXT,"
+        "metadata TEXT,"
+        "date_created TEXT DEFAULT CURRENT_TIMESTAMP,"
+        "date_updated TEXT DEFAULT CURRENT_TIMESTAMP,"
+        "UNIQUE(engine_name, variant));");
+
+    // CTL_INF("Database migration up completed successfully.");
+    return true;
+  } catch (const std::exception& e) {
+    CTL_WRN("Migration up failed: " << e.what());
+    return cpp::fail(e.what());
+  }
+};
+
+inline cpp::result<bool, std::string> MigrateDBDown(SQLite::Database& db) {
+  try {
+    // models
+    {
+      SQLite::Statement query(db,
+                              "SELECT name FROM sqlite_master WHERE "
+                              "type='table' AND name='models'");
+      auto table_exists = query.executeStep();
+      if (table_exists) {
+        // Create a new table with the old schema
+        db.exec(
+            "CREATE TABLE models_old ("
+            "model_id TEXT PRIMARY KEY,"
+            "author_repo_id TEXT,"
+            "branch_name TEXT,"
+            "path_to_model_yaml TEXT,"
+            "model_alias TEXT"
+            ")");
+
+        // Copy data from the current table to the new table
+        db.exec(
+            "INSERT INTO models_old (model_id, author_repo_id, branch_name, "
+            "path_to_model_yaml, model_alias) "
+            "SELECT model_id, author_repo_id, branch_name, path_to_model_yaml, "
+            "model_alias FROM models");
+
+        // Drop the current table
+        db.exec("DROP TABLE models");
+
+        // Rename the new table to the original name
+        db.exec("ALTER TABLE models_old RENAME TO models");
+      }
+    }
+
+    // hardware
+    {
+      SQLite::Statement query(db,
+                              "SELECT name FROM sqlite_master WHERE "
+                              "type='table' AND name='hardware'");
+      auto table_exists = query.executeStep();
+      if (table_exists) {
+        // Create a new table with the old schema
+        db.exec(
+            "CREATE TABLE hardware_old ("
+            "uuid TEXT PRIMARY KEY, "
+            "type TEXT NOT NULL, "
+            "hardware_id INTEGER NOT NULL, "
+            "software_id INTEGER NOT NULL, "
+            "activated INTEGER NOT NULL CHECK (activated IN (0, 1))"
+            ")");
+
+        // Copy data from the current table to the new table
+        db.exec(
+            "INSERT INTO hardware_old (uuid, type, hardware_id, "
+            "software_id, activated) "
+            "SELECT uuid, type, hardware_id, software_id, "
+            "activated FROM hardware");
+
+        // Drop the current table
+        db.exec("DROP TABLE hardware");
+
+        // Rename the new table to the original name
+        db.exec("ALTER TABLE hardware_old RENAME TO hardware");
+      }
+    }
+
+    // engines
+    db.exec("DROP TABLE IF EXISTS engines;");
+    // CTL_INF("Migration down completed successfully.");
+    return true;
+  } catch (const std::exception& e) {
+    CTL_WRN("Migration down failed: " << e.what());
+    return cpp::fail(e.what());
+  }
+}
+
+};  // namespace cortex::migr::v2
\ No newline at end of file

From 80bd6b4b6f778d324740b6f82b34077bb683cb30 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Wed, 4 Dec 2024 13:51:20 +0700
Subject: [PATCH 03/21] fix: add priority

---
 engine/database/hardware.cc         | 36 +++++++++++-----------
 engine/database/hardware.h          | 19 ++++++------
 engine/services/hardware_service.cc | 47 ++++++++++++++++++++---------
 3 files changed, 61 insertions(+), 41 deletions(-)

diff --git a/engine/database/hardware.cc b/engine/database/hardware.cc
index ee68749d5..a1a230ed5 100644
--- a/engine/database/hardware.cc
+++ b/engine/database/hardware.cc
@@ -4,16 +4,14 @@
 
 namespace cortex::db {
 
-Hardwares::Hardwares() : db_(cortex::db::Database::GetInstance().db()) {
-}
+Hardware::Hardware() : db_(cortex::db::Database::GetInstance().db()) {}
 
-Hardwares::Hardwares(SQLite::Database& db) : db_(db) {
-}
+Hardware::Hardware(SQLite::Database& db) : db_(db) {}
 
-Hardwares::~Hardwares() {}
+Hardware::~Hardware() {}
 
 cpp::result<std::vector<HardwareEntry>, std::string>
-Hardwares::LoadHardwareList() const {
+Hardware::LoadHardwareList() const {
   try {
     db_.exec("BEGIN TRANSACTION;");
     cortex::utils::ScopeExit se([this] { db_.exec("COMMIT;"); });
@@ -21,7 +19,7 @@ Hardwares::LoadHardwareList() const {
     SQLite::Statement query(
         db_,
         "SELECT uuid, type, "
-        "hardware_id, software_id, activated FROM hardware");
+        "hardware_id, software_id, activated, priority FROM hardware");
 
     while (query.executeStep()) {
       HardwareEntry entry;
@@ -30,6 +28,7 @@ Hardwares::LoadHardwareList() const {
       entry.hardware_id = query.getColumn(2).getInt();
       entry.software_id = query.getColumn(3).getInt();
       entry.activated = query.getColumn(4).getInt();
+      entry.priority = query.getColumn(5).getInt();
       entries.push_back(entry);
     }
     return entries;
@@ -38,19 +37,20 @@ Hardwares::LoadHardwareList() const {
     return cpp::fail(e.what());
   }
 }
-cpp::result<bool, std::string> Hardwares::AddHardwareEntry(
+cpp::result<bool, std::string> Hardware::AddHardwareEntry(
     const HardwareEntry& new_entry) {
   try {
     SQLite::Statement insert(
         db_,
         "INSERT INTO hardware (uuid, type, "
-        "hardware_id, software_id, activated) VALUES (?, ?, "
-        "?, ?, ?)");
+        "hardware_id, software_id, activated, priority) VALUES (?, ?, "
+        "?, ?, ?, ?)");
     insert.bind(1, new_entry.uuid);
     insert.bind(2, new_entry.type);
     insert.bind(3, new_entry.hardware_id);
     insert.bind(4, new_entry.software_id);
     insert.bind(5, new_entry.activated);
+    insert.bind(6, new_entry.priority);
     insert.exec();
     CTL_INF("Inserted: " << new_entry.ToJsonString());
     return true;
@@ -59,17 +59,19 @@ cpp::result<bool, std::string> Hardwares::AddHardwareEntry(
     return cpp::fail(e.what());
   }
 }
-cpp::result<bool, std::string> Hardwares::UpdateHardwareEntry(
+cpp::result<bool, std::string> Hardware::UpdateHardwareEntry(
     const std::string& id, const HardwareEntry& updated_entry) {
   try {
-    SQLite::Statement upd(db_,
-                          "UPDATE hardware "
-                          "SET hardware_id = ?, software_id = ?, activated = ? "
-                          "WHERE uuid = ?");
+    SQLite::Statement upd(
+        db_,
+        "UPDATE hardware "
+        "SET hardware_id = ?, software_id = ?, activated = ?, priority = ? "
+        "WHERE uuid = ?");
     upd.bind(1, updated_entry.hardware_id);
     upd.bind(2, updated_entry.software_id);
     upd.bind(3, updated_entry.activated);
-    upd.bind(4, id);
+    upd.bind(4, updated_entry.priority);
+    upd.bind(5, id);
     if (upd.exec() == 1) {
       CTL_INF("Updated: " << updated_entry.ToJsonString());
       return true;
@@ -80,7 +82,7 @@ cpp::result<bool, std::string> Hardwares::UpdateHardwareEntry(
   }
 }
 
-cpp::result<bool, std::string> Hardwares::DeleteHardwareEntry(
+cpp::result<bool, std::string> Hardware::DeleteHardwareEntry(
     const std::string& id) {
   try {
     SQLite::Statement del(db_, "DELETE from hardware WHERE uuid = ?");
diff --git a/engine/database/hardware.h b/engine/database/hardware.h
index 0966d58a3..04d0bbda1 100644
--- a/engine/database/hardware.h
+++ b/engine/database/hardware.h
@@ -4,8 +4,8 @@
 #include <trantor/utils/Logger.h>
 #include <string>
 #include <vector>
-#include "utils/result.hpp"
 #include "utils/json_helper.h"
+#include "utils/result.hpp"
 
 namespace cortex::db {
 struct HardwareEntry {
@@ -14,6 +14,7 @@ struct HardwareEntry {
   int hardware_id;
   int software_id;
   bool activated;
+  int priority;
   std::string ToJsonString() const {
     Json::Value root;
     root["uuid"] = uuid;
@@ -21,26 +22,26 @@ struct HardwareEntry {
     root["hardware_id"] = hardware_id;
     root["software_id"] = software_id;
     root["activated"] = activated;
+    root["priority"] = priority;
     return json_helper::DumpJsonString(root);
   }
 };
 
-class Hardwares {
+class Hardware {
 
  private:
   SQLite::Database& db_;
 
-
  public:
-  Hardwares();
-  Hardwares(SQLite::Database& db);
-  ~Hardwares();
+  Hardware();
+  Hardware(SQLite::Database& db);
+  ~Hardware();
 
   cpp::result<std::vector<HardwareEntry>, std::string> LoadHardwareList() const;
-  cpp::result<bool, std::string> AddHardwareEntry(const HardwareEntry& new_entry);
+  cpp::result<bool, std::string> AddHardwareEntry(
+      const HardwareEntry& new_entry);
   cpp::result<bool, std::string> UpdateHardwareEntry(
       const std::string& id, const HardwareEntry& updated_entry);
-  cpp::result<bool, std::string> DeleteHardwareEntry(
-      const std::string& id);
+  cpp::result<bool, std::string> DeleteHardwareEntry(const std::string& id);
 };
 }  // namespace cortex::db
\ No newline at end of file
diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc
index 5574472bd..629d98c82 100644
--- a/engine/services/hardware_service.cc
+++ b/engine/services/hardware_service.cc
@@ -34,7 +34,7 @@ bool TryConnectToServer(const std::string& host, int port) {
 
 HardwareInfo HardwareService::GetHardwareInfo() {
   // append active state
-  cortex::db::Hardwares hw_db;
+  cortex::db::Hardware hw_db;
   auto gpus = cortex::hw::GetGPUInfo();
   auto res = hw_db.LoadHardwareList();
   if (res.has_value()) {
@@ -210,12 +210,21 @@ bool HardwareService::SetActivateHardwareConfig(
     const cortex::hw::ActivateHardwareConfig& ahc) {
   // Note: need to map software_id and hardware_id
   // Update to db
-  cortex::db::Hardwares hw_db;
+  cortex::db::Hardware hw_db;
   // copy all gpu information to new vector
   auto ahc_gpus = ahc.gpus;
-  auto activate = [&ahc_gpus](int software_id) {
-    return std::count(ahc_gpus.begin(), ahc_gpus.end(), software_id) > 0;
+  auto activate = [&ahc](int software_id) {
+    return std::count(ahc.gpus.begin(), ahc.gpus.end(), software_id) > 0;
   };
+  auto priority = [&ahc](int software_id) -> int {
+    for (size_t i = 0; i < ahc.gpus.size(); i++) {
+      if (ahc.gpus[i] == software_id)
+        return i;
+      break;
+    }
+    return INT_MAX;
+  };
+  
   auto res = hw_db.LoadHardwareList();
   if (res.has_value()) {
     bool need_update = false;
@@ -245,6 +254,7 @@ bool HardwareService::SetActivateHardwareConfig(
     // Need to update, proceed
     for (auto& e : res.value()) {
       e.activated = activate(e.software_id);
+      e.priority = priority(e.software_id);
       auto res = hw_db.UpdateHardwareEntry(e.uuid, e);
       if (res.has_error()) {
         CTL_WRN(res.error());
@@ -258,14 +268,14 @@ bool HardwareService::SetActivateHardwareConfig(
 void HardwareService::UpdateHardwareInfos() {
   using HwEntry = cortex::db::HardwareEntry;
   auto gpus = cortex::hw::GetGPUInfo();
-  cortex::db::Hardwares hw_db;
+  cortex::db::Hardware hw_db;
   auto b = hw_db.LoadHardwareList();
-  std::vector<int> activated_gpu_bf;
+  std::vector<std::pair<int, int>> activated_gpu_bf;
   std::string debug_b;
   for (auto const& he : b.value()) {
     if (he.type == "gpu" && he.activated) {
       debug_b += std::to_string(he.software_id) + " ";
-      activated_gpu_bf.push_back(he.software_id);
+      activated_gpu_bf.push_back(std::pair(he.software_id, he.priority));
     }
   }
   CTL_INF("Activated GPUs before: " << debug_b);
@@ -276,7 +286,8 @@ void HardwareService::UpdateHardwareInfos() {
                                               .type = "gpu",
                                               .hardware_id = std::stoi(gpu.id),
                                               .software_id = std::stoi(gpu.id),
-                                              .activated = true});
+                                              .activated = true,
+                                              .priority = INT_MAX});
     if (res.has_error()) {
       CTL_WRN(res.error());
     }
@@ -284,24 +295,26 @@ void HardwareService::UpdateHardwareInfos() {
 
   auto a = hw_db.LoadHardwareList();
   std::vector<HwEntry> a_gpu;
-  std::vector<int> activated_gpu_af;
+  std::vector<std::pair<int, int>> activated_gpu_af;
   std::string debug_a;
   for (auto const& he : a.value()) {
     if (he.type == "gpu" && he.activated) {
       debug_a += std::to_string(he.software_id) + " ";
-      activated_gpu_af.push_back(he.software_id);
+      activated_gpu_af.push_back(std::pair(he.software_id, he.priority));
     }
   }
   CTL_INF("Activated GPUs after: " << debug_a);
   // if hardware list changes, need to restart
-  std::sort(activated_gpu_bf.begin(), activated_gpu_bf.end());
-  std::sort(activated_gpu_af.begin(), activated_gpu_af.end());
+  std::sort(activated_gpu_bf.begin(), activated_gpu_bf.end(),
+            [](auto& p1, auto& p2) { return p1.second < p2.second; });
+  std::sort(activated_gpu_af.begin(), activated_gpu_af.end(),
+            [](auto& p1, auto& p2) { return p1.second < p2.second; });
   bool need_restart = false;
   if (activated_gpu_bf.size() != activated_gpu_af.size()) {
     need_restart = true;
   } else {
     for (size_t i = 0; i < activated_gpu_bf.size(); i++) {
-      if (activated_gpu_bf[i] != activated_gpu_af[i]) {
+      if (activated_gpu_bf[i].first != activated_gpu_af[i].first) {
         need_restart = true;
         break;
       }
@@ -322,7 +335,11 @@ void HardwareService::UpdateHardwareInfos() {
 
   if (need_restart) {
     CTL_INF("Need restart");
-    ahc_ = {.gpus = activated_gpu_af};
+    std::vector<int> gpus;
+    for (auto const& p : activated_gpu_af) {
+      gpus.push_back(p.first);
+    }
+    ahc_ = {.gpus = gpus};
   }
 }
 
@@ -330,7 +347,7 @@ bool HardwareService::IsValidConfig(
     const cortex::hw::ActivateHardwareConfig& ahc) {
   if (ahc.gpus.empty())
     return true;
-  cortex::db::Hardwares hw_db;
+  cortex::db::Hardware hw_db;
   auto is_valid = [&ahc](int software_id) {
     return std::count(ahc.gpus.begin(), ahc.gpus.end(), software_id) > 0;
   };

From 50b774ed1515f7b9ac8c7e486861e438abef580c Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Thu, 5 Dec 2024 09:45:59 +0700
Subject: [PATCH 04/21] fix: db

---
 engine/migrations/v2/migration.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/engine/migrations/v2/migration.h b/engine/migrations/v2/migration.h
index 11bd0298d..d28020645 100644
--- a/engine/migrations/v2/migration.h
+++ b/engine/migrations/v2/migration.h
@@ -192,7 +192,9 @@ inline cpp::result<bool, std::string> MigrateDBDown(SQLite::Database& db) {
     }
 
     // engines
-    db.exec("DROP TABLE IF EXISTS engines;");
+    {
+      // do nothing
+    }
     // CTL_INF("Migration down completed successfully.");
     return true;
   } catch (const std::exception& e) {

From fb581dac10008cbb374b4966d43f6f20112f8eb8 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Thu, 5 Dec 2024 10:48:42 +0700
Subject: [PATCH 05/21] fix: more

---
 engine/services/hardware_service.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc
index 629d98c82..25be78873 100644
--- a/engine/services/hardware_service.cc
+++ b/engine/services/hardware_service.cc
@@ -224,15 +224,15 @@ bool HardwareService::SetActivateHardwareConfig(
     }
     return INT_MAX;
   };
-  
+
   auto res = hw_db.LoadHardwareList();
   if (res.has_value()) {
     bool need_update = false;
-    std::vector<int> activated_ids;
+    std::vector<std::pair<int, int>> activated_ids;
     // Check if need to update
     for (auto const& e : res.value()) {
       if (e.activated) {
-        activated_ids.push_back(e.software_id);
+        activated_ids.push_back(std::pair(e.software_id, e.priority));
       }
     }
     std::sort(activated_ids.begin(), activated_ids.end());
@@ -241,8 +241,11 @@ bool HardwareService::SetActivateHardwareConfig(
       need_update = true;
     } else {
       for (size_t i = 0; i < ahc_gpus.size(); i++) {
-        if (ahc_gpus[i] != activated_ids[i])
+        // if activated id or priority changes
+        if (ahc_gpus[i] != activated_ids[i].first ||
+            i != activated_ids[i].second)
           need_update = true;
+        break;
       }
     }
 

From a66173fba64c68fa07812679069d9f6dafdcfc61 Mon Sep 17 00:00:00 2001
From: vansangpfiev <vansangpfiev@gmail.com>
Date: Thu, 5 Dec 2024 17:35:28 +0700
Subject: [PATCH 06/21] feat: model sources

---
 engine/services/model_source_service.cc | 168 ++++++++++++++++++++++++
 engine/services/model_source_service.h  |  38 ++++++
 engine/utils/json_parser_utils.h        |   2 +-
 3 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 engine/services/model_source_service.cc
 create mode 100644 engine/services/model_source_service.h

diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
new file mode 100644
index 000000000..41224d0f1
--- /dev/null
+++ b/engine/services/model_source_service.cc
@@ -0,0 +1,168 @@
+#include "model_source_service.h"
+#include "database/models.h"
+#include "utils/curl_utils.h"
+#include "utils/huggingface_utils.h"
+#include "utils/logging_utils.h"
+#include "utils/string_utils.h"
+#include "utils/url_parser.h"
+
+namespace services {
+namespace hu = huggingface_utils;
+
+namespace {
+struct ModelInfo {
+  std::string id;
+  int likes;
+  int trending_score;
+  bool is_private;
+  int downloads;
+  std::vector<std::string> tags;
+  std::string created_at;
+  std::string model_id;
+};
+
+std::vector<ModelInfo> ParseJsonString(const std::string& json_str) {
+  std::vector<ModelInfo> models;
+
+  // Parse the JSON string
+  Json::Value root;
+  Json::Reader reader;
+  bool parsing_successful = reader.parse(json_str, root);
+
+  if (!parsing_successful) {
+    std::cerr << "Failed to parse JSON" << std::endl;
+    return models;
+  }
+
+  // Iterate over the JSON array
+  for (const auto& model : root) {
+    ModelInfo info;
+    info.id = model["id"].asString();
+    info.likes = model["likes"].asInt();
+    info.trending_score = model["trendingScore"].asInt();
+    info.is_private = model["private"].asBool();
+    info.downloads = model["downloads"].asInt();
+
+    const Json::Value& tags = model["tags"];
+    for (const auto& tag : tags) {
+      info.tags.push_back(tag.asString());
+    }
+
+    info.created_at = model["createdAt"].asString();
+    info.model_id = model["modelId"].asString();
+    models.push_back(info);
+  }
+
+  return models;
+}
+
+}  // namespace
+cpp::result<bool, std::string> ModelSourceService::AddModelSource(
+    const std::string& model_source) {
+  // https://huggingface.co/Orenguteng
+  // https://huggingface.co/Orenguteng/Llama-3.1-8B-Lexi-Uncensored-V2-GGUF
+  auto res = url_parser::FromUrlString(model_source);
+  if (res.has_error()) {
+    return cpp::fail(res.error());
+  } else {
+    auto& r = res.value();
+    if (r.pathParams.empty() || r.pathParams.size() > 2) {
+      return cpp::fail("Invalid model source url: " + model_source);
+    }
+
+    // Org
+    if (r.pathParams.size() == 1) {
+      // Get model id
+      // for loop, add
+      auto& author = r.pathParams[0];
+      if (author == "cortexso") {
+
+      } else {
+        if (auto res = curl_utils::SimpleGet(
+                "https://huggingface.co/api/models?author=" + author);
+            res.has_value()) {
+          auto models = ParseJsonString(res.value());
+          for (auto const& m : models) {
+            CTL_INF(m.id);
+            auto author_model = string_utils::SplitBy(m.id, "/");
+            if (author_model.size() == 2) {
+              auto const& author = author_model[0];
+              auto const& model_name = author_model[1];
+              AddRepo(model_source, author, model_name);
+            }
+          }
+        }
+      }
+
+    } else {  // Repo
+      if (r.pathParams[0] == "cortexso") {
+
+      } else {
+        auto const& author = r.pathParams[0];
+        auto const& model_name = r.pathParams[1];
+        if (auto res = AddRepo(model_source, author, model_name);
+            res.has_error()) {
+          return cpp::fail(res.error());
+        }
+      }
+    }
+  }
+  return true;
+}
+
+cpp::result<bool, std::string> ModelSourceService::RemoveModelSource(
+    const std::string& model_source) {
+  return true;
+}
+
+cpp::result<bool, std::string> ModelSourceService::AddOrg(
+    const std::string& org) {
+  return true;
+}
+
+cpp::result<bool, std::string> ModelSourceService::AddRepo(
+    const std::string& model_source, const std::string& author,
+    const std::string& model_name) {
+  auto repo_info = hu::GetHuggingFaceModelRepoInfo(author, model_name);
+  if (repo_info.has_error()) {
+    return cpp::fail(repo_info.error());
+  }
+
+  if (!repo_info->gguf.has_value()) {
+    return cpp::fail(
+        "Not a GGUF model. Currently, only GGUF single file is "
+        "supported.");
+  }
+
+  for (const auto& sibling : repo_info->siblings) {
+    if (string_utils::EndsWith(sibling.rfilename, ".gguf")) {
+      cortex::db::Models model_db;
+      std::string model_id =
+          author + ":" + model_name + ":" + sibling.rfilename;
+      if (!model_db.HasModel(model_id)) {
+        cortex::db::ModelEntry e = {
+            .model = model_id,
+            .author_repo_id = author,
+            .branch_name = "main",
+            .path_to_model_yaml = "",
+            .model_alias = "",
+            .model_format = "hf-gguf",
+            .model_source = model_source,
+            .status = cortex::db::ModelStatus::Undownloaded,
+            .engine = "llama-cpp"};
+        model_db.AddModelEntry(e);
+      }
+    }
+  }
+  return true;
+}
+
+cpp::result<bool, std::string> ModelSourceService::RemoveOrg(
+    const std::string& org) {
+  return true;
+}
+cpp::result<bool, std::string> ModelSourceService::RemoveRepo(
+    const std::string& repo) {
+  return true;
+}
+}  // namespace services
\ No newline at end of file
diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h
new file mode 100644
index 000000000..22a2d2ef9
--- /dev/null
+++ b/engine/services/model_source_service.h
@@ -0,0 +1,38 @@
+#pragma once
+#include "utils/result.hpp"
+
+// struct ModelEntry {
+//   std::string model;
+//   std::string author_repo_id;
+//   std::string branch_name;
+//   std::string path_to_model_yaml;
+//   std::string model_alias;
+//   std::string model_format;
+//   std::string model_source;
+//   ModelStatus status;
+//   std::string engine;
+// };
+namespace services {
+class ModelSourceService {
+ public:
+  // model source can be HF organization, repo or others (for example Modelscope,..)
+  // default is HF, need to check if it is organization or repo
+  // if repo:
+  // if org: api/models?author=cortexso
+  cpp::result<bool, std::string> AddModelSource(
+      const std::string& model_source);
+
+  cpp::result<bool, std::string> RemoveModelSource(
+      const std::string& model_source);
+
+ private:
+  // models database
+  cpp::result<bool, std::string> AddOrg(const std::string& org);
+  cpp::result<bool, std::string> AddRepo(const std::string& model_source,
+                                         const std::string& author,
+                                         const std::string& model_name);
+
+  cpp::result<bool, std::string> RemoveOrg(const std::string& org);
+  cpp::result<bool, std::string> RemoveRepo(const std::string& repo);
+};
+}  // namespace services
\ No newline at end of file
diff --git a/engine/utils/json_parser_utils.h b/engine/utils/json_parser_utils.h
index 3ebd2c546..b4ea1a7e1 100644
--- a/engine/utils/json_parser_utils.h
+++ b/engine/utils/json_parser_utils.h
@@ -10,7 +10,7 @@ template <typename T>
 T jsonToValue(const Json::Value& value);
 
 template <>
-std::string jsonToValue(const Json::Value& value) {
+inline std::string jsonToValue(const Json::Value& value) {
   return value.asString();
 }
 

From aebc317c1ab5c14d74707024126d37054cee3ddc Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Fri, 6 Dec 2024 13:43:46 +0700
Subject: [PATCH 07/21] feat: support delete API

---
 engine/cli/command_line_parser.cc        |   5 +-
 engine/cli/command_line_parser.h         |   1 +
 engine/cli/commands/model_list_cmd.cc    |  78 +++++++----
 engine/cli/commands/model_list_cmd.h     |   3 +-
 engine/controllers/models.cc             |  72 +++++++++-
 engine/controllers/models.h              |  20 ++-
 engine/database/models.cc                | 123 +++++++----------
 engine/database/models.h                 |  22 ++-
 engine/main.cc                           |   5 +-
 engine/services/model_service.cc         |   5 +-
 engine/services/model_source_service.cc  | 168 +++++++++++++++++++++--
 engine/services/model_source_service.h   |   4 +
 engine/test/components/test_models_db.cc |  20 ---
 13 files changed, 376 insertions(+), 150 deletions(-)

diff --git a/engine/cli/command_line_parser.cc b/engine/cli/command_line_parser.cc
index 9d5d83ffc..eb943d6f8 100644
--- a/engine/cli/command_line_parser.cc
+++ b/engine/cli/command_line_parser.cc
@@ -253,6 +253,8 @@ void CommandLineParser::SetupModelCommands() {
                             "Display cpu mode");
   list_models_cmd->add_flag("--gpu_mode", cml_data_.display_gpu_mode,
                             "Display gpu mode");
+  list_models_cmd->add_flag("--remote", cml_data_.display_available_model,
+                            "Display available models to download");
   list_models_cmd->group(kSubcommands);
   list_models_cmd->callback([this]() {
     if (std::exchange(executed_, true))
@@ -261,7 +263,8 @@ void CommandLineParser::SetupModelCommands() {
         cml_data_.config.apiServerHost,
         std::stoi(cml_data_.config.apiServerPort), cml_data_.filter,
         cml_data_.display_engine, cml_data_.display_version,
-        cml_data_.display_cpu_mode, cml_data_.display_gpu_mode);
+        cml_data_.display_cpu_mode, cml_data_.display_gpu_mode,
+        cml_data_.display_available_model);
   });
 
   auto get_models_cmd =
diff --git a/engine/cli/command_line_parser.h b/engine/cli/command_line_parser.h
index aec10dcb4..f99f01ae6 100644
--- a/engine/cli/command_line_parser.h
+++ b/engine/cli/command_line_parser.h
@@ -66,6 +66,7 @@ class CommandLineParser {
     bool display_version = false;
     bool display_cpu_mode = false;
     bool display_gpu_mode = false;
+    bool display_available_model = false;
     std::string filter = "";
     std::string log_level = "INFO";
 
diff --git a/engine/cli/commands/model_list_cmd.cc b/engine/cli/commands/model_list_cmd.cc
index 7990563f3..f33667f1f 100644
--- a/engine/cli/commands/model_list_cmd.cc
+++ b/engine/cli/commands/model_list_cmd.cc
@@ -21,7 +21,7 @@ using Row_t =
 void ModelListCmd::Exec(const std::string& host, int port,
                         const std::string& filter, bool display_engine,
                         bool display_version, bool display_cpu_mode,
-                        bool display_gpu_mode) {
+                        bool display_gpu_mode, bool is_remote) {
   // Start server if server is not started yet
   if (!commands::IsServerAlive(host, port)) {
     CLI_LOG("Starting server ...");
@@ -73,40 +73,62 @@ void ModelListCmd::Exec(const std::string& host, int port,
         continue;
       }
 
-      count += 1;
+      if (is_remote) {
+        if (v["status"].asString() != "undownloaded") {
+          continue;
+        }
 
-      std::vector<std::string> row = {std::to_string(count),
-                                      v["model"].asString()};
-      if (display_engine) {
-        row.push_back(v["engine"].asString());
-      }
-      if (display_version) {
-        row.push_back(v["version"].asString());
-      }
+        count += 1;
 
-      if (auto& r = v["recommendation"]; !r.isNull()) {
-        if (display_cpu_mode) {
-          if (!r["cpu_mode"].isNull()) {
-            row.push_back("RAM: " + r["cpu_mode"]["ram"].asString() + " MiB");
-          }
+        std::vector<std::string> row = {std::to_string(count),
+                                        v["model"].asString()};
+        if (display_engine) {
+          row.push_back(v["engine"].asString());
+        }
+        if (display_version) {
+          row.push_back(v["version"].asString());
+        }
+        table.add_row({row.begin(), row.end()});
+      } else {
+        if (v["status"].asString() == "undownloaded") {
+          continue;
+        }
+
+        count += 1;
+
+        std::vector<std::string> row = {std::to_string(count),
+                                        v["model"].asString()};
+        if (display_engine) {
+          row.push_back(v["engine"].asString());
+        }
+        if (display_version) {
+          row.push_back(v["version"].asString());
         }
 
-        if (display_gpu_mode) {
-          if (!r["gpu_mode"].isNull()) {
-            std::string s;
-            s += "ngl: " + r["gpu_mode"][0]["ngl"].asString() + " - ";
-            s += "context: " + r["gpu_mode"][0]["context_length"].asString() +
-                 " - ";
-            s += "RAM: " + r["gpu_mode"][0]["ram"].asString() + " MiB - ";
-            s += "VRAM: " + r["gpu_mode"][0]["vram"].asString() + " MiB - ";
-            s += "recommended ngl: " +
-                 r["gpu_mode"][0]["recommend_ngl"].asString();
-            row.push_back(s);
+        if (auto& r = v["recommendation"]; !r.isNull()) {
+          if (display_cpu_mode) {
+            if (!r["cpu_mode"].isNull()) {
+              row.push_back("RAM: " + r["cpu_mode"]["ram"].asString() + " MiB");
+            }
+          }
+
+          if (display_gpu_mode) {
+            if (!r["gpu_mode"].isNull()) {
+              std::string s;
+              s += "ngl: " + r["gpu_mode"][0]["ngl"].asString() + " - ";
+              s += "context: " + r["gpu_mode"][0]["context_length"].asString() +
+                   " - ";
+              s += "RAM: " + r["gpu_mode"][0]["ram"].asString() + " MiB - ";
+              s += "VRAM: " + r["gpu_mode"][0]["vram"].asString() + " MiB - ";
+              s += "recommended ngl: " +
+                   r["gpu_mode"][0]["recommend_ngl"].asString();
+              row.push_back(s);
+            }
           }
         }
-      }
 
-      table.add_row({row.begin(), row.end()});
+        table.add_row({row.begin(), row.end()});
+      }
     }
   }
 
diff --git a/engine/cli/commands/model_list_cmd.h b/engine/cli/commands/model_list_cmd.h
index 791c1ecf6..839888803 100644
--- a/engine/cli/commands/model_list_cmd.h
+++ b/engine/cli/commands/model_list_cmd.h
@@ -8,6 +8,7 @@ class ModelListCmd {
  public:
   void Exec(const std::string& host, int port, const std::string& filter,
             bool display_engine = false, bool display_version = false,
-            bool display_cpu_mode = false, bool display_gpu_mode = false);
+            bool display_cpu_mode = false, bool display_gpu_mode = false,
+            bool is_remote = false);
 };
 }  // namespace commands
diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc
index de14886da..47592261a 100644
--- a/engine/controllers/models.cc
+++ b/engine/controllers/models.cc
@@ -172,6 +172,26 @@ void Models::ListModel(
   if (list_entry) {
     for (const auto& model_entry : list_entry.value()) {
       try {
+        if (model_entry.status == cortex::db::ModelStatus::Undownloaded) {
+          Json::Value obj;
+          obj["id"] = model_entry.model;
+          obj["model"] = model_entry.model;
+          auto status_to_string = [](cortex::db::ModelStatus status) {
+            switch (status) {
+              case cortex::db::ModelStatus::Remote:
+                return "remote";
+              case cortex::db::ModelStatus::Downloaded:
+                return "downloaded";
+              case cortex::db::ModelStatus::Undownloaded:
+                return "undownloaded";
+            }
+            return "unknown";
+          };
+          obj["status"] = status_to_string(model_entry.status);
+          obj["engine"] = model_entry.engine;
+          data.append(std::move(obj));
+          continue;
+        }
         yaml_handler.ModelConfigFromFile(
             fmu::ToAbsoluteCortexDataPath(
                 fs::path(model_entry.path_to_model_yaml))
@@ -182,7 +202,7 @@ void Models::ListModel(
           Json::Value obj = model_config.ToJson();
           obj["id"] = model_entry.model;
           obj["model"] = model_entry.model;
-          obj["model"] = model_entry.model;
+          obj["status"] = "downloaded";
           auto es = model_service_->GetEstimation(model_entry.model);
           if (es.has_value()) {
             obj["recommendation"] = hardware::ToJson(es.value());
@@ -723,4 +743,54 @@ void Models::AddRemoteModel(
     resp->setStatusCode(k400BadRequest);
     callback(resp);
   }
+}
+
+void Models::AddModelSource(
+    const HttpRequestPtr& req,
+    std::function<void(const HttpResponsePtr&)>&& callback) {
+  if (!http_util::HasFieldInReq(req, callback, "source")) {
+    return;
+  }
+
+  auto model_source = (*(req->getJsonObject())).get("source", "").asString();
+  auto res = model_src_svc_->AddModelSource(model_source);
+  if (res.has_error()) {
+    Json::Value ret;
+    ret["message"] = res.error();
+    auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
+    resp->setStatusCode(k400BadRequest);
+    callback(resp);
+  } else {
+    auto const& info = res.value();
+    Json::Value ret;
+    ret["message"] = "Model source is added successfully!";
+    auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
+    resp->setStatusCode(k200OK);
+    callback(resp);
+  }
+}
+
+void Models::DeleteModelSource(
+    const HttpRequestPtr& req,
+    std::function<void(const HttpResponsePtr&)>&& callback) {
+  if (!http_util::HasFieldInReq(req, callback, "source")) {
+    return;
+  }
+
+  auto model_source = (*(req->getJsonObject())).get("source", "").asString();
+  auto res = model_src_svc_->RemoveModelSource(model_source);
+  if (res.has_error()) {
+    Json::Value ret;
+    ret["message"] = res.error();
+    auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
+    resp->setStatusCode(k400BadRequest);
+    callback(resp);
+  } else {
+    auto const& info = res.value();
+    Json::Value ret;
+    ret["message"] = "Model source is deleted successfully!";
+    auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
+    resp->setStatusCode(k200OK);
+    callback(resp);
+  }
 }
\ No newline at end of file
diff --git a/engine/controllers/models.h b/engine/controllers/models.h
index b2b288adc..1efec2a4b 100644
--- a/engine/controllers/models.h
+++ b/engine/controllers/models.h
@@ -4,6 +4,7 @@
 #include <trantor/utils/Logger.h>
 #include "services/engine_service.h"
 #include "services/model_service.h"
+#include "services/model_source_service.h"
 
 using namespace drogon;
 
@@ -23,6 +24,8 @@ class Models : public drogon::HttpController<Models, false> {
   METHOD_ADD(Models::GetModelStatus, "/status/{1}", Get);
   METHOD_ADD(Models::AddRemoteModel, "/add", Options, Post);
   METHOD_ADD(Models::GetRemoteModels, "/remote/{1}", Get);
+  METHOD_ADD(Models::AddModelSource, "/sources", Post);
+  METHOD_ADD(Models::DeleteModelSource, "/sources", Delete);
 
   ADD_METHOD_TO(Models::PullModel, "/v1/models/pull", Options, Post);
   ADD_METHOD_TO(Models::AbortPullModel, "/v1/models/pull", Options, Delete);
@@ -36,11 +39,16 @@ class Models : public drogon::HttpController<Models, false> {
   ADD_METHOD_TO(Models::GetModelStatus, "/v1/models/status/{1}", Get);
   ADD_METHOD_TO(Models::AddRemoteModel, "/v1/models/add", Options, Post);
   ADD_METHOD_TO(Models::GetRemoteModels, "/v1/models/remote/{1}", Get);
+  ADD_METHOD_TO(Models::AddModelSource, "/v1/models/sources", Post);
+  ADD_METHOD_TO(Models::DeleteModelSource, "/v1/models/sources", Delete);
   METHOD_LIST_END
 
   explicit Models(std::shared_ptr<ModelService> model_service,
-                  std::shared_ptr<EngineService> engine_service)
-      : model_service_{model_service}, engine_service_{engine_service} {}
+                  std::shared_ptr<EngineService> engine_service,
+                  std::shared_ptr<services::ModelSourceService> mss)
+      : model_service_{model_service},
+        engine_service_{engine_service},
+        model_src_svc_(mss) {}
 
   void PullModel(const HttpRequestPtr& req,
                  std::function<void(const HttpResponsePtr&)>&& callback);
@@ -84,7 +92,15 @@ class Models : public drogon::HttpController<Models, false> {
                        std::function<void(const HttpResponsePtr&)>&& callback,
                        const std::string& engine_id);
 
+  void AddModelSource(const HttpRequestPtr& req,
+                      std::function<void(const HttpResponsePtr&)>&& callback);
+
+  void DeleteModelSource(
+      const HttpRequestPtr& req,
+      std::function<void(const HttpResponsePtr&)>&& callback);
+
  private:
   std::shared_ptr<ModelService> model_service_;
   std::shared_ptr<EngineService> engine_service_;
+  std::shared_ptr<services::ModelSourceService> model_src_svc_;
 };
diff --git a/engine/database/models.cc b/engine/database/models.cc
index fb2128396..8e2baa822 100644
--- a/engine/database/models.cc
+++ b/engine/database/models.cc
@@ -49,13 +49,10 @@ cpp::result<std::vector<ModelEntry>, std::string> Models::LoadModelList()
 }
 
 bool Models::IsUnique(const std::vector<ModelEntry>& entries,
-                      const std::string& model_id,
-                      const std::string& model_alias) const {
+                      const std::string& model_id) const {
   return std::none_of(
-      entries.begin(), entries.end(), [&](const ModelEntry& entry) {
-        return entry.model == model_id || entry.model_alias == model_id ||
-               entry.model == model_alias || entry.model_alias == model_alias;
-      });
+      entries.begin(), entries.end(),
+      [&](const ModelEntry& entry) { return entry.model == model_id; });
 }
 
 cpp::result<std::vector<ModelEntry>, std::string> Models::LoadModelListNoLock()
@@ -87,66 +84,6 @@ cpp::result<std::vector<ModelEntry>, std::string> Models::LoadModelListNoLock()
   }
 }
 
-std::string Models::GenerateShortenedAlias(
-    const std::string& model_id, const std::vector<ModelEntry>& entries) const {
-  std::vector<std::string> parts;
-  std::istringstream iss(model_id);
-  std::string part;
-  while (std::getline(iss, part, ':')) {
-    parts.push_back(part);
-  }
-
-  if (parts.empty()) {
-    return model_id;  // Return original if no parts
-  }
-
-  // Extract the filename without extension
-  std::string filename = parts.back();
-  size_t last_dot_pos = filename.find_last_of('.');
-  if (last_dot_pos != std::string::npos) {
-    filename = filename.substr(0, last_dot_pos);
-  }
-
-  // Convert to lowercase
-  std::transform(filename.begin(), filename.end(), filename.begin(),
-                 [](unsigned char c) { return std::tolower(c); });
-
-  // Generate alias candidates
-  std::vector<std::string> candidates;
-  candidates.push_back(filename);
-
-  if (parts.size() >= 2) {
-    candidates.push_back(parts[parts.size() - 2] + ":" + filename);
-  }
-
-  if (parts.size() >= 3) {
-    candidates.push_back(parts[parts.size() - 3] + ":" +
-                         parts[parts.size() - 2] + ":" + filename);
-  }
-
-  if (parts.size() >= 4) {
-    candidates.push_back(parts[0] + ":" + parts[1] + ":" +
-                         parts[parts.size() - 2] + ":" + filename);
-  }
-
-  // Find the first unique candidate
-  for (const auto& candidate : candidates) {
-    if (IsUnique(entries, model_id, candidate)) {
-      return candidate;
-    }
-  }
-
-  // If all candidates are taken, append a number to the last candidate
-  std::string base_candidate = candidates.back();
-  int suffix = 1;
-  std::string unique_candidate = base_candidate;
-  while (!IsUnique(entries, model_id, unique_candidate)) {
-    unique_candidate = base_candidate + "-" + std::to_string(suffix++);
-  }
-
-  return unique_candidate;
-}
-
 cpp::result<ModelEntry, std::string> Models::GetModelInfo(
     const std::string& identifier) const {
   try {
@@ -190,8 +127,7 @@ void Models::PrintModelInfo(const ModelEntry& entry) const {
   LOG_INFO << "Engine: " << entry.engine;
 }
 
-cpp::result<bool, std::string> Models::AddModelEntry(ModelEntry new_entry,
-                                                     bool use_short_alias) {
+cpp::result<bool, std::string> Models::AddModelEntry(ModelEntry new_entry) {
   try {
     db_.exec("BEGIN TRANSACTION;");
     cortex::utils::ScopeExit se([this] { db_.exec("COMMIT;"); });
@@ -200,11 +136,7 @@ cpp::result<bool, std::string> Models::AddModelEntry(ModelEntry new_entry,
       CTL_WRN(model_list.error());
       return cpp::fail(model_list.error());
     }
-    if (IsUnique(model_list.value(), new_entry.model, new_entry.model_alias)) {
-      if (use_short_alias) {
-        new_entry.model_alias =
-            GenerateShortenedAlias(new_entry.model, model_list.value());
-      }
+    if (IsUnique(model_list.value(), new_entry.model)) {
 
       SQLite::Statement insert(
           db_,
@@ -271,7 +203,7 @@ cpp::result<bool, std::string> Models::UpdateModelAlias(
       return cpp::fail(model_list.error());
     }
     // Check new_model_alias is unique
-    if (IsUnique(model_list.value(), new_model_alias, new_model_alias)) {
+    if (IsUnique(model_list.value(), new_model_alias)) {
       SQLite::Statement upd(db_,
                             "UPDATE models "
                             "SET model_alias = ? "
@@ -305,6 +237,32 @@ cpp::result<bool, std::string> Models::DeleteModelEntry(
   }
 }
 
+cpp::result<bool, std::string> Models::DeleteModelEntryWithOrg(
+    const std::string& src) {
+  try {
+    SQLite::Statement del(db_,
+                          "DELETE from models WHERE model_source LIKE ? AND "
+                          "status = \"undownloaded\"");
+    del.bind(1, src + "%");
+    return del.exec() == 1;
+  } catch (const std::exception& e) {
+    return cpp::fail(e.what());
+  }
+}
+
+cpp::result<bool, std::string> Models::DeleteModelEntryWithRepo(
+    const std::string& src) {
+  try {
+    SQLite::Statement del(db_,
+                          "DELETE from models WHERE model_source = ? AND "
+                          "status = \"undownloaded\"");
+    del.bind(1, src);
+    return del.exec() == 1;
+  } catch (const std::exception& e) {
+    return cpp::fail(e.what());
+  }
+}
+
 cpp::result<std::vector<std::string>, std::string> Models::FindRelatedModel(
     const std::string& identifier) const {
   try {
@@ -339,4 +297,21 @@ bool Models::HasModel(const std::string& identifier) const {
   }
 }
 
+cpp::result<std::vector<std::string>, std::string> Models::GetModelSources()
+    const {
+  try {
+    std::vector<std::string> sources;
+    SQLite::Statement query(db_,
+                            "SELECT DISTINCT model_source FROM models WHERE "
+                            "status = \"undownloaded\"");
+
+    while (query.executeStep()) {
+      sources.push_back(query.getColumn(0).getString());
+    }
+    return sources;
+  } catch (const std::exception& e) {
+    return cpp::fail(e.what());
+  }
+}
+
 }  // namespace cortex::db
\ No newline at end of file
diff --git a/engine/database/models.h b/engine/database/models.h
index dd6e2a5a1..6a66e9c8a 100644
--- a/engine/database/models.h
+++ b/engine/database/models.h
@@ -8,14 +8,10 @@
 
 namespace cortex::db {
 
-enum class ModelStatus {
-  Remote,
-  Downloaded,
-  Undownloaded
-};
+enum class ModelStatus { Remote, Downloaded, Undownloaded };
 
 struct ModelEntry {
-  std::string model;  
+  std::string model;
   std::string author_repo_id;
   std::string branch_name;
   std::string path_to_model_yaml;
@@ -32,8 +28,7 @@ class Models {
   SQLite::Database& db_;
 
   bool IsUnique(const std::vector<ModelEntry>& entries,
-                const std::string& model_id,
-                const std::string& model_alias) const;
+                const std::string& model_id) const;
 
   cpp::result<std::vector<ModelEntry>, std::string> LoadModelListNoLock() const;
 
@@ -45,23 +40,24 @@ class Models {
   Models();
   Models(SQLite::Database& db);
   ~Models();
-  std::string GenerateShortenedAlias(
-      const std::string& model_id,
-      const std::vector<ModelEntry>& entries) const;
   cpp::result<ModelEntry, std::string> GetModelInfo(
       const std::string& identifier) const;
   void PrintModelInfo(const ModelEntry& entry) const;
-  cpp::result<bool, std::string> AddModelEntry(ModelEntry new_entry,
-                                               bool use_short_alias = false);
+  cpp::result<bool, std::string> AddModelEntry(ModelEntry new_entry);
   cpp::result<bool, std::string> UpdateModelEntry(
       const std::string& identifier, const ModelEntry& updated_entry);
   cpp::result<bool, std::string> DeleteModelEntry(
       const std::string& identifier);
+  cpp::result<bool, std::string> DeleteModelEntryWithOrg(
+      const std::string& src);
+  cpp::result<bool, std::string> DeleteModelEntryWithRepo(
+      const std::string& src);
   cpp::result<bool, std::string> UpdateModelAlias(
       const std::string& model_id, const std::string& model_alias);
   cpp::result<std::vector<std::string>, std::string> FindRelatedModel(
       const std::string& identifier) const;
   bool HasModel(const std::string& identifier) const;
+  cpp::result<std::vector<std::string>, std::string> GetModelSources() const;
 };
 
 }  // namespace cortex::db
\ No newline at end of file
diff --git a/engine/main.cc b/engine/main.cc
index 0177a2143..1e977d9d4 100644
--- a/engine/main.cc
+++ b/engine/main.cc
@@ -18,6 +18,7 @@
 #include "services/file_watcher_service.h"
 #include "services/message_service.h"
 #include "services/model_service.h"
+#include "services/model_source_service.h"
 #include "services/thread_service.h"
 #include "utils/archive_utils.h"
 #include "utils/cortex_utils.h"
@@ -134,6 +135,7 @@ void RunServer(std::optional<int> port, bool ignore_cout) {
   auto engine_service = std::make_shared<EngineService>(download_service);
   auto inference_svc =
       std::make_shared<services::InferenceService>(engine_service);
+  auto model_src_svc = std::make_shared<services::ModelSourceService>();
   auto model_service = std::make_shared<ModelService>(
       download_service, inference_svc, engine_service);
 
@@ -145,7 +147,8 @@ void RunServer(std::optional<int> port, bool ignore_cout) {
   auto thread_ctl = std::make_shared<Threads>(thread_srv, message_srv);
   auto message_ctl = std::make_shared<Messages>(message_srv);
   auto engine_ctl = std::make_shared<Engines>(engine_service);
-  auto model_ctl = std::make_shared<Models>(model_service, engine_service);
+  auto model_ctl =
+      std::make_shared<Models>(model_service, engine_service, model_src_svc);
   auto event_ctl = std::make_shared<Events>(event_queue_ptr);
   auto pm_ctl = std::make_shared<ProcessManager>();
   auto hw_ctl = std::make_shared<Hardware>(engine_service, hw_service);
diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc
index d81a9b649..7c4e21d40 100644
--- a/engine/services/model_service.cc
+++ b/engine/services/model_service.cc
@@ -71,7 +71,7 @@ void ParseGguf(const DownloadItem& ggufDownloadItem,
       .path_to_model_yaml = rel.string(),
       .model_alias = ggufDownloadItem.id,
       .status = cortex::db::ModelStatus::Downloaded};
-  auto result = modellist_utils_obj.AddModelEntry(model_entry, true);
+  auto result = modellist_utils_obj.AddModelEntry(model_entry);
   if (result.has_error()) {
     CTL_WRN("Error adding model to modellist: " + result.error());
   }
@@ -136,6 +136,9 @@ void ModelService::ForceIndexingModelList() {
 
   CTL_DBG("Database model size: " + std::to_string(list_entry.value().size()));
   for (const auto& model_entry : list_entry.value()) {
+    if(model_entry.status != cortex::db::ModelStatus::Downloaded) {
+      continue;
+    }
     try {
       yaml_handler.ModelConfigFromFile(
           fmu::ToAbsoluteCortexDataPath(
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index 41224d0f1..fcf777895 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -1,4 +1,5 @@
 #include "model_source_service.h"
+#include <unordered_set>
 #include "database/models.h"
 #include "utils/curl_utils.h"
 #include "utils/huggingface_utils.h"
@@ -70,13 +71,32 @@ cpp::result<bool, std::string> ModelSourceService::AddModelSource(
       return cpp::fail("Invalid model source url: " + model_source);
     }
 
-    // Org
-    if (r.pathParams.size() == 1) {
-      // Get model id
-      // for loop, add
+    if (auto is_org = r.pathParams.size() == 1; is_org) {
       auto& author = r.pathParams[0];
       if (author == "cortexso") {
-
+        if (auto res = curl_utils::SimpleGet(
+                "https://huggingface.co/api/models?author=" + author);
+            res.has_value()) {
+          auto models = ParseJsonString(res.value());
+          for (auto const& m : models) {
+            CTL_INF(m.id);
+            auto author_model = string_utils::SplitBy(m.id, "/");
+            if (author_model.size() == 2) {
+              auto const& author = author_model[0];
+              auto const& model_name = author_model[1];
+              auto branches = huggingface_utils::GetModelRepositoryBranches(
+                  "cortexso", model_name);
+              if (branches.has_error()) {
+                CTL_INF(branches.error());
+                continue;
+              }
+              for (auto const& [branch, _] : branches.value()) {
+                CTL_INF(branch);
+                AddCortexsoRepoBranch(model_source, author, model_name, branch);
+              }
+            }
+          }
+        }
       } else {
         if (auto res = curl_utils::SimpleGet(
                 "https://huggingface.co/api/models?author=" + author);
@@ -95,11 +115,19 @@ cpp::result<bool, std::string> ModelSourceService::AddModelSource(
       }
 
     } else {  // Repo
+      auto const& author = r.pathParams[0];
+      auto const& model_name = r.pathParams[1];
       if (r.pathParams[0] == "cortexso") {
-
+        auto branches = huggingface_utils::GetModelRepositoryBranches(
+            "cortexso", model_name);
+        if (branches.has_error()) {
+          return cpp::fail(branches.error());
+        }
+        for (auto const& [branch, _] : branches.value()) {
+          CTL_INF(branch);
+          AddCortexsoRepoBranch(model_source, author, model_name, branch);
+        }
       } else {
-        auto const& author = r.pathParams[0];
-        auto const& model_name = r.pathParams[1];
         if (auto res = AddRepo(model_source, author, model_name);
             res.has_error()) {
           return cpp::fail(res.error());
@@ -112,6 +140,30 @@ cpp::result<bool, std::string> ModelSourceService::AddModelSource(
 
 cpp::result<bool, std::string> ModelSourceService::RemoveModelSource(
     const std::string& model_source) {
+  CTL_INF("Remove model source: " << model_source);
+  auto res = url_parser::FromUrlString(model_source);
+  if (res.has_error()) {
+    return cpp::fail(res.error());
+  } else {
+    auto& r = res.value();
+    if (r.pathParams.empty() || r.pathParams.size() > 2) {
+      return cpp::fail("Invalid model source url: " + model_source);
+    }
+    cortex::db::Models model_db;
+    if (r.pathParams.size() == 1) {
+      if (auto del_res = model_db.DeleteModelEntryWithOrg(model_source);
+          del_res.has_error()) {
+        CTL_INF(del_res.error());
+        return cpp::fail(del_res.error());
+      }
+    } else {
+      if (auto del_res = model_db.DeleteModelEntryWithRepo(model_source);
+          del_res.has_error()) {
+        CTL_INF(del_res.error());
+        return cpp::fail(del_res.error());
+      }
+    }
+  }
   return true;
 }
 
@@ -134,6 +186,7 @@ cpp::result<bool, std::string> ModelSourceService::AddRepo(
         "supported.");
   }
 
+
   for (const auto& sibling : repo_info->siblings) {
     if (string_utils::EndsWith(sibling.rfilename, ".gguf")) {
       cortex::db::Models model_db;
@@ -151,7 +204,61 @@ cpp::result<bool, std::string> ModelSourceService::AddRepo(
             .status = cortex::db::ModelStatus::Undownloaded,
             .engine = "llama-cpp"};
         model_db.AddModelEntry(e);
+      } else {
+        // update
+      }
+    }
+  }
+  // delete old data
+  return true;
+}
+
+cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepoBranch(
+    const std::string& model_source, const std::string& author,
+    const std::string& model_name, const std::string& branch) {
+
+  url_parser::Url url = {
+      .protocol = "https",
+      .host = kHuggingFaceHost,
+      .pathParams = {"api", "models", "cortexso", model_name, "tree", branch},
+  };
+
+  auto result = curl_utils::SimpleGetJson(url.ToFullPath());
+  if (result.has_error()) {
+    return cpp::fail("Model " + model_name + " not found");
+  }
+
+  bool has_gguf = false;
+  for (const auto& value : result.value()) {
+    auto path = value["path"].asString();
+    if (path.find(".gguf") != std::string::npos) {
+      has_gguf = true;
+    }
+  }
+  if (!has_gguf) {
+    CTL_INF("Only support gguf file format! - branch: " << branch);
+    return false;
+  } else {
+    cortex::db::Models model_db;
+    std::string model_id = model_name + ":" + branch;
+    if (!model_db.HasModel(model_id)) {
+      CTL_INF("Adding model to db: " << model_name << ":" << branch);
+      cortex::db::ModelEntry e = {
+          .model = model_id,
+          .author_repo_id = author,
+          .branch_name = branch,
+          .path_to_model_yaml = "",
+          .model_alias = "",
+          .model_format = "cortexso",
+          .model_source = model_source,
+          .status = cortex::db::ModelStatus::Undownloaded,
+          .engine = "llama-cpp"};
+      if (auto res = model_db.AddModelEntry(e);
+          res.has_error() || !res.value()) {
+        CTL_DBG("Cannot add model to db: " << model_id);
       }
+    } else {
+      CTL_DBG("Model exists: " << model_id);
     }
   }
   return true;
@@ -165,4 +272,49 @@ cpp::result<bool, std::string> ModelSourceService::RemoveRepo(
     const std::string& repo) {
   return true;
 }
+
+void ModelSourceService::SyncModelSource() {
+  cortex::db::Models model_db;
+  auto res = model_db.GetModelSources();
+  if (res.has_error()) {
+    CTL_INF(res.error());
+  } else {
+    for (auto const& src : res.value()) {
+      CTL_INF(src);
+    }
+
+    std::unordered_set<std::string> orgs;
+    std::vector<std::string> repos;
+    for (auto const& src : res.value()) {
+      auto url_res = url_parser::FromUrlString(src);
+      if (url_res.has_value()) {
+        if (url_res->pathParams.size() == 1) {
+          orgs.insert(src);
+        } else if (url_res->pathParams.size() == 2) {
+          repos.push_back(src);
+        }
+      }
+    }
+
+    // Get list to update
+    std::vector<std::string> update_cand(orgs.begin(), orgs.end());
+    auto get_org = [](const std::string& rp) {
+      return rp.substr(0, rp.find_last_of("/"));
+    };
+    for (auto const& repo : repos) {
+      if (orgs.find(get_org(repo)) != orgs.end()) {
+        update_cand.push_back(repo);
+      }
+    }
+
+    // Sync cortex.db with the upstream data
+    // add new model if it does not exist
+    // update model if it does exist
+    // delete model if remote has removed model
+    for (auto const& c : update_cand) {
+      AddModelSource(c);
+    }
+  }
+}
+
 }  // namespace services
\ No newline at end of file
diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h
index 22a2d2ef9..c717eadba 100644
--- a/engine/services/model_source_service.h
+++ b/engine/services/model_source_service.h
@@ -31,8 +31,12 @@ class ModelSourceService {
   cpp::result<bool, std::string> AddRepo(const std::string& model_source,
                                          const std::string& author,
                                          const std::string& model_name);
+  cpp::result<bool, std::string> AddCortexsoRepoBranch(
+      const std::string& model_source, const std::string& author,
+      const std::string& model_name, const std::string& branch);
 
   cpp::result<bool, std::string> RemoveOrg(const std::string& org);
   cpp::result<bool, std::string> RemoveRepo(const std::string& repo);
+  void SyncModelSource();
 };
 }  // namespace services
\ No newline at end of file
diff --git a/engine/test/components/test_models_db.cc b/engine/test/components/test_models_db.cc
index ab0ea9f70..bce72a084 100644
--- a/engine/test/components/test_models_db.cc
+++ b/engine/test/components/test_models_db.cc
@@ -104,26 +104,6 @@ TEST_F(ModelsTestSuite, TestDeleteModelEntry) {
   EXPECT_TRUE(model_list_.GetModelInfo(kTestModel.model).has_error());
 }
 
-TEST_F(ModelsTestSuite, TestGenerateShortenedAlias) {
-  EXPECT_TRUE(model_list_.AddModelEntry(kTestModel).value());
-  auto models1 = model_list_.LoadModelList();
-  auto alias = model_list_.GenerateShortenedAlias(
-      "huggingface.co:bartowski:llama3.1-7b-gguf:Model_ID_Xxx.gguf",
-      models1.value());
-  EXPECT_EQ(alias, "model_id_xxx");
-  EXPECT_TRUE(model_list_.UpdateModelAlias(kTestModel.model, alias).value());
-
-  // Test with existing entries to force longer alias
-  auto models2 = model_list_.LoadModelList();
-  alias = model_list_.GenerateShortenedAlias(
-      "huggingface.co:bartowski:llama3.1-7b-gguf:Model_ID_Xxx.gguf",
-      models2.value());
-  EXPECT_EQ(alias, "llama3.1-7b-gguf:model_id_xxx");
-
-  // Clean up
-  EXPECT_TRUE(model_list_.DeleteModelEntry(kTestModel.model).value());
-}
-
 TEST_F(ModelsTestSuite, TestPersistence) {
   EXPECT_TRUE(model_list_.AddModelEntry(kTestModel).value());
 

From 1eac21dc53c707f470bc84c8c79b97758215a661 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Fri, 6 Dec 2024 13:59:16 +0700
Subject: [PATCH 08/21] feat: cli: support models sources add

---
 engine/cli/command_line_parser.cc           | 35 +++++++++++++++++
 engine/cli/command_line_parser.h            |  1 +
 engine/cli/commands/model_source_add_cmd.cc | 43 +++++++++++++++++++++
 engine/cli/commands/model_source_add_cmd.h  | 12 ++++++
 4 files changed, 91 insertions(+)
 create mode 100644 engine/cli/commands/model_source_add_cmd.cc
 create mode 100644 engine/cli/commands/model_source_add_cmd.h

diff --git a/engine/cli/command_line_parser.cc b/engine/cli/command_line_parser.cc
index eb943d6f8..ff478bad4 100644
--- a/engine/cli/command_line_parser.cc
+++ b/engine/cli/command_line_parser.cc
@@ -20,6 +20,7 @@
 #include "commands/model_import_cmd.h"
 #include "commands/model_list_cmd.h"
 #include "commands/model_pull_cmd.h"
+#include "commands/model_source_add_cmd.h"
 #include "commands/model_start_cmd.h"
 #include "commands/model_stop_cmd.h"
 #include "commands/model_upd_cmd.h"
@@ -332,6 +333,40 @@ void CommandLineParser::SetupModelCommands() {
                                     std::stoi(cml_data_.config.apiServerPort),
                                     cml_data_.model_id, cml_data_.model_path);
   });
+
+  auto model_source_cmd = models_cmd->add_subcommand(
+      "sources", "Subcommands for managing model sources");
+  model_source_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
+                          " models sources [options] [subcommand]");
+  model_source_cmd->group(kSubcommands);
+
+  model_source_cmd->callback([this, model_source_cmd] {
+    if (std::exchange(executed_, true))
+      return;
+    if (model_source_cmd->get_subcommands().empty()) {
+      CLI_LOG(model_source_cmd->help());
+    }
+  });
+
+  auto model_src_add_cmd =
+      model_source_cmd->add_subcommand("add", "Add a model source");
+  model_src_add_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
+                           " models sources add [model_source]");
+  model_src_add_cmd->group(kSubcommands);
+  model_src_add_cmd->add_option("source", cml_data_.model_src, "");
+  model_src_add_cmd->callback([&]() {
+    if (std::exchange(executed_, true))
+      return;
+    if (cml_data_.model_src.empty()) {
+      CLI_LOG("[model_source] is required\n");
+      CLI_LOG(model_src_add_cmd->help());
+      return;
+    };
+
+    commands::ModelSourceAddCmd().Exec(
+        cml_data_.config.apiServerHost,
+        std::stoi(cml_data_.config.apiServerPort), cml_data_.model_src);
+  });
 }
 
 void CommandLineParser::SetupConfigsCommands() {
diff --git a/engine/cli/command_line_parser.h b/engine/cli/command_line_parser.h
index f99f01ae6..896c026d0 100644
--- a/engine/cli/command_line_parser.h
+++ b/engine/cli/command_line_parser.h
@@ -75,6 +75,7 @@ class CommandLineParser {
     int port;
     config_yaml_utils::CortexConfig config;
     std::unordered_map<std::string, std::string> model_update_options;
+    std::string model_src;
   };
   CmlData cml_data_;
   std::unordered_map<std::string, std::string> config_update_opts_;
diff --git a/engine/cli/commands/model_source_add_cmd.cc b/engine/cli/commands/model_source_add_cmd.cc
new file mode 100644
index 000000000..796359fdb
--- /dev/null
+++ b/engine/cli/commands/model_source_add_cmd.cc
@@ -0,0 +1,43 @@
+#include "model_source_add_cmd.h"
+#include "cortex_upd_cmd.h"
+#include "hardware_activate_cmd.h"
+#include "run_cmd.h"
+#include "server_start_cmd.h"
+#include "utils/cli_selection_utils.h"
+#include "utils/json_helper.h"
+#include "utils/logging_utils.h"
+
+namespace commands {
+bool ModelSourceAddCmd::Exec(const std::string& host, int port, const std::string& model_source) {
+  // Start server if server is not started yet
+  if (!commands::IsServerAlive(host, port)) {
+    CLI_LOG("Starting server ...");
+    commands::ServerStartCmd ssc;
+    if (!ssc.Exec(host, port)) {
+      return false;
+    }
+  }
+
+  auto url = url_parser::Url{
+      .protocol = "http",
+      .host = host + ":" + std::to_string(port),
+      .pathParams = {"v1", "models", "sources"},
+  };
+
+  Json::Value json_data;
+  json_data["source"] = model_source;
+
+  auto data_str = json_data.toStyledString();
+  auto res = curl_utils::SimplePostJson(url.ToFullPath(), data_str);
+  if (res.has_error()) {
+    auto root = json_helper::ParseJsonString(res.error());
+    CLI_LOG(root["message"].asString());
+    return false;
+  }
+
+  CLI_LOG("Added model source: " << model_source);
+  return true;
+}
+
+
+};  // namespace commands
diff --git a/engine/cli/commands/model_source_add_cmd.h b/engine/cli/commands/model_source_add_cmd.h
new file mode 100644
index 000000000..6d3bcc6c0
--- /dev/null
+++ b/engine/cli/commands/model_source_add_cmd.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+namespace commands {
+
+class ModelSourceAddCmd {
+ public:
+  bool Exec(const std::string& host, int port, const std::string& model_source);
+};
+}  // namespace commands

From 9160a094172859837f4c5897c2b8bd0e4fe1c8db Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Fri, 6 Dec 2024 14:17:08 +0700
Subject: [PATCH 09/21] feat: cli: model source delete

---
 engine/cli/command_line_parser.cc           | 21 +++++++++++
 engine/cli/commands/model_source_add_cmd.cc |  5 ---
 engine/cli/commands/model_source_del_cmd.cc | 39 +++++++++++++++++++++
 engine/cli/commands/model_source_del_cmd.h  | 12 +++++++
 engine/services/model_source_service.cc     | 13 +++++--
 5 files changed, 83 insertions(+), 7 deletions(-)
 create mode 100644 engine/cli/commands/model_source_del_cmd.cc
 create mode 100644 engine/cli/commands/model_source_del_cmd.h

diff --git a/engine/cli/command_line_parser.cc b/engine/cli/command_line_parser.cc
index ff478bad4..e143c2412 100644
--- a/engine/cli/command_line_parser.cc
+++ b/engine/cli/command_line_parser.cc
@@ -21,6 +21,7 @@
 #include "commands/model_list_cmd.h"
 #include "commands/model_pull_cmd.h"
 #include "commands/model_source_add_cmd.h"
+#include "commands/model_source_del_cmd.h"
 #include "commands/model_start_cmd.h"
 #include "commands/model_stop_cmd.h"
 #include "commands/model_upd_cmd.h"
@@ -367,6 +368,26 @@ void CommandLineParser::SetupModelCommands() {
         cml_data_.config.apiServerHost,
         std::stoi(cml_data_.config.apiServerPort), cml_data_.model_src);
   });
+
+  auto model_src_del_cmd =
+      model_source_cmd->add_subcommand("remove", "Remove a model source");
+  model_src_del_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
+                           " models sources remove [model_source]");
+  model_src_del_cmd->group(kSubcommands);
+  model_src_del_cmd->add_option("source", cml_data_.model_src, "");
+  model_src_del_cmd->callback([&]() {
+    if (std::exchange(executed_, true))
+      return;
+    if (cml_data_.model_src.empty()) {
+      CLI_LOG("[model_source] is required\n");
+      CLI_LOG(model_src_del_cmd->help());
+      return;
+    };
+
+    commands::ModelSourceDelCmd().Exec(
+        cml_data_.config.apiServerHost,
+        std::stoi(cml_data_.config.apiServerPort), cml_data_.model_src);
+  });
 }
 
 void CommandLineParser::SetupConfigsCommands() {
diff --git a/engine/cli/commands/model_source_add_cmd.cc b/engine/cli/commands/model_source_add_cmd.cc
index 796359fdb..2fadbe8ec 100644
--- a/engine/cli/commands/model_source_add_cmd.cc
+++ b/engine/cli/commands/model_source_add_cmd.cc
@@ -1,12 +1,7 @@
 #include "model_source_add_cmd.h"
-#include "cortex_upd_cmd.h"
-#include "hardware_activate_cmd.h"
-#include "run_cmd.h"
 #include "server_start_cmd.h"
-#include "utils/cli_selection_utils.h"
 #include "utils/json_helper.h"
 #include "utils/logging_utils.h"
-
 namespace commands {
 bool ModelSourceAddCmd::Exec(const std::string& host, int port, const std::string& model_source) {
   // Start server if server is not started yet
diff --git a/engine/cli/commands/model_source_del_cmd.cc b/engine/cli/commands/model_source_del_cmd.cc
new file mode 100644
index 000000000..c3c1694e7
--- /dev/null
+++ b/engine/cli/commands/model_source_del_cmd.cc
@@ -0,0 +1,39 @@
+#include "model_source_del_cmd.h"
+#include "server_start_cmd.h"
+#include "utils/json_helper.h"
+#include "utils/logging_utils.h"
+
+namespace commands {
+bool ModelSourceDelCmd::Exec(const std::string& host, int port, const std::string& model_source) {
+  // Start server if server is not started yet
+  if (!commands::IsServerAlive(host, port)) {
+    CLI_LOG("Starting server ...");
+    commands::ServerStartCmd ssc;
+    if (!ssc.Exec(host, port)) {
+      return false;
+    }
+  }
+
+  auto url = url_parser::Url{
+      .protocol = "http",
+      .host = host + ":" + std::to_string(port),
+      .pathParams = {"v1", "models", "sources"},
+  };
+
+  Json::Value json_data;
+  json_data["source"] = model_source;
+
+  auto data_str = json_data.toStyledString();
+  auto res = curl_utils::SimpleDeleteJson(url.ToFullPath(), data_str);
+  if (res.has_error()) {
+    auto root = json_helper::ParseJsonString(res.error());
+    CLI_LOG(root["message"].asString());
+    return false;
+  }
+
+  CLI_LOG("Removed model source: " << model_source);
+  return true;
+}
+
+
+};  // namespace commands
diff --git a/engine/cli/commands/model_source_del_cmd.h b/engine/cli/commands/model_source_del_cmd.h
new file mode 100644
index 000000000..5015a609a
--- /dev/null
+++ b/engine/cli/commands/model_source_del_cmd.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+namespace commands {
+
+class ModelSourceDelCmd {
+ public:
+  bool Exec(const std::string& host, int port, const std::string& model_source);
+};
+}  // namespace commands
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index fcf777895..d40286bc1 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -140,6 +140,16 @@ cpp::result<bool, std::string> ModelSourceService::AddModelSource(
 
 cpp::result<bool, std::string> ModelSourceService::RemoveModelSource(
     const std::string& model_source) {
+  cortex::db::Models model_db;
+  auto srcs = model_db.GetModelSources();
+  if (srcs.has_error()) {
+    return cpp::fail(srcs.error());
+  } else {
+    auto& v = srcs.value();
+    if (std::find(v.begin(), v.end(), model_source) == v.end()) {
+      return cpp::fail("Model source does not exist: " + model_source);
+    }
+  }
   CTL_INF("Remove model source: " << model_source);
   auto res = url_parser::FromUrlString(model_source);
   if (res.has_error()) {
@@ -149,7 +159,7 @@ cpp::result<bool, std::string> ModelSourceService::RemoveModelSource(
     if (r.pathParams.empty() || r.pathParams.size() > 2) {
       return cpp::fail("Invalid model source url: " + model_source);
     }
-    cortex::db::Models model_db;
+
     if (r.pathParams.size() == 1) {
       if (auto del_res = model_db.DeleteModelEntryWithOrg(model_source);
           del_res.has_error()) {
@@ -186,7 +196,6 @@ cpp::result<bool, std::string> ModelSourceService::AddRepo(
         "supported.");
   }
 
-
   for (const auto& sibling : repo_info->siblings) {
     if (string_utils::EndsWith(sibling.rfilename, ".gguf")) {
       cortex::db::Models model_db;

From 4c49f06d94eb8fe657a3727351478c8351e9f474 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Fri, 6 Dec 2024 15:20:23 +0700
Subject: [PATCH 10/21] feat: cli: add model source list

---
 engine/cli/command_line_parser.cc            | 15 ++++++
 engine/cli/commands/model_source_list_cmd.cc | 55 ++++++++++++++++++++
 engine/cli/commands/model_source_list_cmd.h  | 11 ++++
 engine/controllers/models.cc                 | 24 +++++++++
 engine/controllers/models.h                  |  5 ++
 engine/services/model_source_service.cc      |  6 +++
 engine/services/model_source_service.h       |  2 +
 7 files changed, 118 insertions(+)
 create mode 100644 engine/cli/commands/model_source_list_cmd.cc
 create mode 100644 engine/cli/commands/model_source_list_cmd.h

diff --git a/engine/cli/command_line_parser.cc b/engine/cli/command_line_parser.cc
index e143c2412..962a13538 100644
--- a/engine/cli/command_line_parser.cc
+++ b/engine/cli/command_line_parser.cc
@@ -22,6 +22,7 @@
 #include "commands/model_pull_cmd.h"
 #include "commands/model_source_add_cmd.h"
 #include "commands/model_source_del_cmd.h"
+#include "commands/model_source_list_cmd.h"
 #include "commands/model_start_cmd.h"
 #include "commands/model_stop_cmd.h"
 #include "commands/model_upd_cmd.h"
@@ -388,6 +389,20 @@ void CommandLineParser::SetupModelCommands() {
         cml_data_.config.apiServerHost,
         std::stoi(cml_data_.config.apiServerPort), cml_data_.model_src);
   });
+
+  auto model_src_list_cmd =
+      model_source_cmd->add_subcommand("list", "List all model sources");
+  model_src_list_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
+                            " models sources list");
+  model_src_list_cmd->group(kSubcommands);
+  model_src_list_cmd->callback([&]() {
+    if (std::exchange(executed_, true))
+      return;
+
+    commands::ModelSourceListCmd().Exec(
+        cml_data_.config.apiServerHost,
+        std::stoi(cml_data_.config.apiServerPort));
+  });
 }
 
 void CommandLineParser::SetupConfigsCommands() {
diff --git a/engine/cli/commands/model_source_list_cmd.cc b/engine/cli/commands/model_source_list_cmd.cc
new file mode 100644
index 000000000..3095777ad
--- /dev/null
+++ b/engine/cli/commands/model_source_list_cmd.cc
@@ -0,0 +1,55 @@
+#include "model_source_list_cmd.h"
+#include <json/reader.h>
+#include <json/value.h>
+#include <iostream>
+#include <vector>
+#include "server_start_cmd.h"
+#include "utils/curl_utils.h"
+#include "utils/json_helper.h"
+#include "utils/logging_utils.h"
+#include "utils/string_utils.h"
+#include "utils/url_parser.h"
+// clang-format off
+#include <tabulate/table.hpp>
+// clang-format on
+
+namespace commands {
+
+bool ModelSourceListCmd::Exec(const std::string& host, int port) {
+  // Start server if server is not started yet
+  if (!commands::IsServerAlive(host, port)) {
+    CLI_LOG("Starting server ...");
+    commands::ServerStartCmd ssc;
+    if (!ssc.Exec(host, port)) {
+      return false;
+    }
+  }
+
+  tabulate::Table table;
+  table.add_row({"#", "Model Source"});
+
+  auto url = url_parser::Url{
+      .protocol = "http",
+      .host = host + ":" + std::to_string(port),
+      .pathParams = {"v1", "models", "sources"},
+  };
+  auto result = curl_utils::SimpleGetJson(url.ToFullPath());
+  if (result.has_error()) {
+    CTL_ERR(result.error());
+    return false;
+  }
+  int count = 0;
+
+  if (!result.value()["data"].isNull()) {
+    for (auto const& v : result.value()["data"]) {
+      auto model_source = v.asString();
+      count += 1;
+      std::vector<std::string> row = {std::to_string(count), model_source};
+      table.add_row({row.begin(), row.end()});
+    }
+  }
+
+  std::cout << table << std::endl;
+  return true;
+}
+};  // namespace commands
diff --git a/engine/cli/commands/model_source_list_cmd.h b/engine/cli/commands/model_source_list_cmd.h
new file mode 100644
index 000000000..99116f592
--- /dev/null
+++ b/engine/cli/commands/model_source_list_cmd.h
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+
+namespace commands {
+
+class ModelSourceListCmd {
+ public:
+  bool Exec(const std::string& host, int port);
+};
+}  // namespace commands
diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc
index 47592261a..d0b420d35 100644
--- a/engine/controllers/models.cc
+++ b/engine/controllers/models.cc
@@ -793,4 +793,28 @@ void Models::DeleteModelSource(
     resp->setStatusCode(k200OK);
     callback(resp);
   }
+}
+
+void Models::GetModelSources(
+    const HttpRequestPtr& req,
+    std::function<void(const HttpResponsePtr&)>&& callback) {
+  auto res = model_src_svc_->GetModelSources();
+  if (res.has_error()) {
+    Json::Value ret;
+    ret["message"] = res.error();
+    auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
+    resp->setStatusCode(k400BadRequest);
+    callback(resp);
+  } else {
+    auto const& info = res.value();
+    Json::Value ret;
+    Json::Value data(Json::arrayValue);
+    for (auto const& i : info) {
+      data.append(i);
+    }
+    ret["data"] = data;
+    auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
+    resp->setStatusCode(k200OK);
+    callback(resp);
+  }
 }
\ No newline at end of file
diff --git a/engine/controllers/models.h b/engine/controllers/models.h
index 1efec2a4b..d3200f33a 100644
--- a/engine/controllers/models.h
+++ b/engine/controllers/models.h
@@ -26,6 +26,7 @@ class Models : public drogon::HttpController<Models, false> {
   METHOD_ADD(Models::GetRemoteModels, "/remote/{1}", Get);
   METHOD_ADD(Models::AddModelSource, "/sources", Post);
   METHOD_ADD(Models::DeleteModelSource, "/sources", Delete);
+  METHOD_ADD(Models::GetModelSources, "/sources", Get);
 
   ADD_METHOD_TO(Models::PullModel, "/v1/models/pull", Options, Post);
   ADD_METHOD_TO(Models::AbortPullModel, "/v1/models/pull", Options, Delete);
@@ -41,6 +42,7 @@ class Models : public drogon::HttpController<Models, false> {
   ADD_METHOD_TO(Models::GetRemoteModels, "/v1/models/remote/{1}", Get);
   ADD_METHOD_TO(Models::AddModelSource, "/v1/models/sources", Post);
   ADD_METHOD_TO(Models::DeleteModelSource, "/v1/models/sources", Delete);
+  ADD_METHOD_TO(Models::GetModelSources, "/v1/models/sources", Get);
   METHOD_LIST_END
 
   explicit Models(std::shared_ptr<ModelService> model_service,
@@ -99,6 +101,9 @@ class Models : public drogon::HttpController<Models, false> {
       const HttpRequestPtr& req,
       std::function<void(const HttpResponsePtr&)>&& callback);
 
+  void GetModelSources(const HttpRequestPtr& req,
+                       std::function<void(const HttpResponsePtr&)>&& callback);
+
  private:
   std::shared_ptr<ModelService> model_service_;
   std::shared_ptr<EngineService> engine_service_;
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index d40286bc1..42788dc7e 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -177,6 +177,12 @@ cpp::result<bool, std::string> ModelSourceService::RemoveModelSource(
   return true;
 }
 
+cpp::result<std::vector<std::string>, std::string>
+ModelSourceService::GetModelSources() {
+  cortex::db::Models model_db;
+  return model_db.GetModelSources();
+}
+
 cpp::result<bool, std::string> ModelSourceService::AddOrg(
     const std::string& org) {
   return true;
diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h
index c717eadba..ebebe78b4 100644
--- a/engine/services/model_source_service.h
+++ b/engine/services/model_source_service.h
@@ -25,6 +25,8 @@ class ModelSourceService {
   cpp::result<bool, std::string> RemoveModelSource(
       const std::string& model_source);
 
+  cpp::result<std::vector<std::string>, std::string> GetModelSources();
+
  private:
   // models database
   cpp::result<bool, std::string> AddOrg(const std::string& org);

From 34d766f12a27a6ffece16ecbb4c84ab62e825622 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Mon, 9 Dec 2024 09:09:34 +0700
Subject: [PATCH 11/21] feat: sync cortex.db

---
 engine/cli/commands/model_source_list_cmd.cc |   1 +
 engine/controllers/models.cc                 |   1 +
 engine/database/models.cc                    |  17 ++
 engine/database/models.h                     |   2 +
 engine/services/model_source_service.cc      | 217 +++++++++++++------
 engine/services/model_source_service.h       |  38 ++--
 6 files changed, 193 insertions(+), 83 deletions(-)

diff --git a/engine/cli/commands/model_source_list_cmd.cc b/engine/cli/commands/model_source_list_cmd.cc
index 3095777ad..ae69c5aef 100644
--- a/engine/cli/commands/model_source_list_cmd.cc
+++ b/engine/cli/commands/model_source_list_cmd.cc
@@ -38,6 +38,7 @@ bool ModelSourceListCmd::Exec(const std::string& host, int port) {
     CTL_ERR(result.error());
     return false;
   }
+  table.format().font_color(tabulate::Color::green);
   int count = 0;
 
   if (!result.value()["data"].isNull()) {
diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc
index d0b420d35..83adc6352 100644
--- a/engine/controllers/models.cc
+++ b/engine/controllers/models.cc
@@ -187,6 +187,7 @@ void Models::ListModel(
             }
             return "unknown";
           };
+          obj["modelSource"] = model_entry.model_source;
           obj["status"] = status_to_string(model_entry.status);
           obj["engine"] = model_entry.engine;
           data.append(std::move(obj));
diff --git a/engine/database/models.cc b/engine/database/models.cc
index 8e2baa822..12587b3dd 100644
--- a/engine/database/models.cc
+++ b/engine/database/models.cc
@@ -314,4 +314,21 @@ cpp::result<std::vector<std::string>, std::string> Models::GetModelSources()
   }
 }
 
+cpp::result<std::vector<std::string>, std::string> Models::GetModels(
+    const std::string& model_src) const {
+  try {
+    std::vector<std::string> ids;
+    SQLite::Statement query(db_,
+                            "SELECT model_id FROM models WHERE model_source = "
+                            "? AND status = \"undownloaded\"");
+    query.bind(1, model_src);
+    while (query.executeStep()) {
+      ids.push_back(query.getColumn(0).getString());
+    }
+    return ids;
+  } catch (const std::exception& e) {
+    return cpp::fail(e.what());
+  }
+}
+
 }  // namespace cortex::db
\ No newline at end of file
diff --git a/engine/database/models.h b/engine/database/models.h
index 6a66e9c8a..79d6d878d 100644
--- a/engine/database/models.h
+++ b/engine/database/models.h
@@ -58,6 +58,8 @@ class Models {
       const std::string& identifier) const;
   bool HasModel(const std::string& identifier) const;
   cpp::result<std::vector<std::string>, std::string> GetModelSources() const;
+  cpp::result<std::vector<std::string>, std::string> GetModels(
+      const std::string& model_src) const;
 };
 
 }  // namespace cortex::db
\ No newline at end of file
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index 42788dc7e..6bfaa7a06 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -1,4 +1,5 @@
 #include "model_source_service.h"
+#include <chrono>
 #include <unordered_set>
 #include "database/models.h"
 #include "utils/curl_utils.h"
@@ -58,6 +59,19 @@ std::vector<ModelInfo> ParseJsonString(const std::string& json_str) {
 }
 
 }  // namespace
+
+ModelSourceService::ModelSourceService() {
+  sync_db_thread_ = std::thread(&ModelSourceService::SyncModelSource, this);
+  running_ = true;
+}
+ModelSourceService::~ModelSourceService() {
+  running_ = false;
+  if (sync_db_thread_.joinable()) {
+    sync_db_thread_.join();
+  }
+  CTL_INF("Done cleanup thread");
+}
+
 cpp::result<bool, std::string> ModelSourceService::AddModelSource(
     const std::string& model_source) {
   // https://huggingface.co/Orenguteng
@@ -78,6 +92,12 @@ cpp::result<bool, std::string> ModelSourceService::AddModelSource(
                 "https://huggingface.co/api/models?author=" + author);
             res.has_value()) {
           auto models = ParseJsonString(res.value());
+          // Get models from db
+          cortex::db::Models model_db;
+
+          auto model_list_before = model_db.GetModels(model_source)
+                                       .value_or(std::vector<std::string>{});
+          std::unordered_set<std::string> updated_model_list;
           for (auto const& m : models) {
             CTL_INF(m.id);
             auto author_model = string_utils::SplitBy(m.id, "/");
@@ -92,23 +112,52 @@ cpp::result<bool, std::string> ModelSourceService::AddModelSource(
               }
               for (auto const& [branch, _] : branches.value()) {
                 CTL_INF(branch);
-                AddCortexsoRepoBranch(model_source, author, model_name, branch);
+                auto add_res = AddCortexsoRepoBranch(model_source, author,
+                                                     model_name, branch)
+                                   .value_or(std::unordered_set<std::string>{});
+                for (auto const& a : add_res) {
+                  updated_model_list.insert(a);
+                }
               }
             }
           }
+          // Clean up
+          for (auto const& mid : model_list_before) {
+            if (updated_model_list.find(mid) == updated_model_list.end()) {
+              model_db.DeleteModelEntry(mid);
+            }
+          }
         }
       } else {
         if (auto res = curl_utils::SimpleGet(
                 "https://huggingface.co/api/models?author=" + author);
             res.has_value()) {
           auto models = ParseJsonString(res.value());
+          // Get models from db
+          cortex::db::Models model_db;
+
+          auto model_list_before = model_db.GetModels(model_source)
+                                       .value_or(std::vector<std::string>{});
+          std::unordered_set<std::string> updated_model_list;
+          // Add new models
           for (auto const& m : models) {
             CTL_INF(m.id);
             auto author_model = string_utils::SplitBy(m.id, "/");
             if (author_model.size() == 2) {
               auto const& author = author_model[0];
               auto const& model_name = author_model[1];
-              AddRepo(model_source, author, model_name);
+              auto add_res = AddRepo(model_source, author, model_name)
+                                 .value_or(std::unordered_set<std::string>{});
+              for (auto const& a : add_res) {
+                updated_model_list.insert(a);
+              }
+            }
+          }
+
+          // Clean up
+          for (auto const& mid : model_list_before) {
+            if (updated_model_list.find(mid) == updated_model_list.end()) {
+              model_db.DeleteModelEntry(mid);
             }
           }
         }
@@ -123,14 +172,46 @@ cpp::result<bool, std::string> ModelSourceService::AddModelSource(
         if (branches.has_error()) {
           return cpp::fail(branches.error());
         }
+        // Get models from db
+        cortex::db::Models model_db;
+
+        auto model_list_before = model_db.GetModels(model_source)
+                                     .value_or(std::vector<std::string>{});
+        std::unordered_set<std::string> updated_model_list;
+
         for (auto const& [branch, _] : branches.value()) {
           CTL_INF(branch);
-          AddCortexsoRepoBranch(model_source, author, model_name, branch);
+          auto add_res =
+              AddCortexsoRepoBranch(model_source, author, model_name, branch)
+                  .value_or(std::unordered_set<std::string>{});
+          for (auto const& a : add_res) {
+            updated_model_list.insert(a);
+          }
+        }
+
+        // Clean up
+        for (auto const& mid : model_list_before) {
+          if (updated_model_list.find(mid) == updated_model_list.end()) {
+            model_db.DeleteModelEntry(mid);
+          }
         }
       } else {
-        if (auto res = AddRepo(model_source, author, model_name);
-            res.has_error()) {
+        // Get models from db
+        cortex::db::Models model_db;
+
+        auto model_list_before = model_db.GetModels(model_source)
+                                     .value_or(std::vector<std::string>{});
+        std::unordered_set<std::string> updated_model_list;
+        auto add_res = AddRepo(model_source, author, model_name);
+        if (res.has_error()) {
           return cpp::fail(res.error());
+        } else {
+          updated_model_list = add_res.value();
+        }
+        for (auto const& mid : model_list_before) {
+          if (updated_model_list.find(mid) == updated_model_list.end()) {
+            model_db.DeleteModelEntry(mid);
+          }
         }
       }
     }
@@ -183,14 +264,11 @@ ModelSourceService::GetModelSources() {
   return model_db.GetModelSources();
 }
 
-cpp::result<bool, std::string> ModelSourceService::AddOrg(
-    const std::string& org) {
-  return true;
-}
-
-cpp::result<bool, std::string> ModelSourceService::AddRepo(
-    const std::string& model_source, const std::string& author,
-    const std::string& model_name) {
+cpp::result<std::unordered_set<std::string>, std::string>
+ModelSourceService::AddRepo(const std::string& model_source,
+                            const std::string& author,
+                            const std::string& model_name) {
+  std::unordered_set<std::string> res;
   auto repo_info = hu::GetHuggingFaceModelRepoInfo(author, model_name);
   if (repo_info.has_error()) {
     return cpp::fail(repo_info.error());
@@ -222,15 +300,19 @@ cpp::result<bool, std::string> ModelSourceService::AddRepo(
       } else {
         // update
       }
+      res.insert(model_id);
     }
   }
-  // delete old data
-  return true;
+
+  return res;
 }
 
-cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepoBranch(
-    const std::string& model_source, const std::string& author,
-    const std::string& model_name, const std::string& branch) {
+cpp::result<std::unordered_set<std::string>, std::string>
+ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source,
+                                          const std::string& author,
+                                          const std::string& model_name,
+                                          const std::string& branch) {
+  std::unordered_set<std::string> res;
 
   url_parser::Url url = {
       .protocol = "https",
@@ -252,7 +334,7 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepoBranch(
   }
   if (!has_gguf) {
     CTL_INF("Only support gguf file format! - branch: " << branch);
-    return false;
+    return {};
   } else {
     cortex::db::Models model_db;
     std::string model_id = model_name + ":" + branch;
@@ -275,59 +357,68 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepoBranch(
     } else {
       CTL_DBG("Model exists: " << model_id);
     }
+    res.insert(model_id);
   }
-  return true;
-}
-
-cpp::result<bool, std::string> ModelSourceService::RemoveOrg(
-    const std::string& org) {
-  return true;
-}
-cpp::result<bool, std::string> ModelSourceService::RemoveRepo(
-    const std::string& repo) {
-  return true;
+  return res;
 }
 
 void ModelSourceService::SyncModelSource() {
-  cortex::db::Models model_db;
-  auto res = model_db.GetModelSources();
-  if (res.has_error()) {
-    CTL_INF(res.error());
-  } else {
-    for (auto const& src : res.value()) {
-      CTL_INF(src);
-    }
+  // Do interval check for 10 minutes
+  constexpr const int kIntervalCheck = 10 * 60;
+  auto start_time = std::chrono::steady_clock::now();
+  while (running_) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(500));
+    auto current_time = std::chrono::steady_clock::now();
+    auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(
+                            current_time - start_time)
+                            .count();
+
+    if (elapsed_time > kIntervalCheck) {
+      CTL_DBG("Start to sync cortex.db");
+      start_time = current_time;
+
+      cortex::db::Models model_db;
+      auto res = model_db.GetModelSources();
+      if (res.has_error()) {
+        CTL_INF(res.error());
+      } else {
+        for (auto const& src : res.value()) {
+          CTL_DBG(src);
+        }
 
-    std::unordered_set<std::string> orgs;
-    std::vector<std::string> repos;
-    for (auto const& src : res.value()) {
-      auto url_res = url_parser::FromUrlString(src);
-      if (url_res.has_value()) {
-        if (url_res->pathParams.size() == 1) {
-          orgs.insert(src);
-        } else if (url_res->pathParams.size() == 2) {
-          repos.push_back(src);
+        std::unordered_set<std::string> orgs;
+        std::vector<std::string> repos;
+        for (auto const& src : res.value()) {
+          auto url_res = url_parser::FromUrlString(src);
+          if (url_res.has_value()) {
+            if (url_res->pathParams.size() == 1) {
+              orgs.insert(src);
+            } else if (url_res->pathParams.size() == 2) {
+              repos.push_back(src);
+            }
+          }
         }
-      }
-    }
 
-    // Get list to update
-    std::vector<std::string> update_cand(orgs.begin(), orgs.end());
-    auto get_org = [](const std::string& rp) {
-      return rp.substr(0, rp.find_last_of("/"));
-    };
-    for (auto const& repo : repos) {
-      if (orgs.find(get_org(repo)) != orgs.end()) {
-        update_cand.push_back(repo);
+        // Get list to update
+        std::vector<std::string> update_cand(orgs.begin(), orgs.end());
+        auto get_org = [](const std::string& rp) {
+          return rp.substr(0, rp.find_last_of("/"));
+        };
+        for (auto const& repo : repos) {
+          if (orgs.find(get_org(repo)) != orgs.end()) {
+            update_cand.push_back(repo);
+          }
+        }
+
+        // Sync cortex.db with the upstream data
+        for (auto const& c : update_cand) {
+          if(auto res = AddModelSource(c); res.has_error()) {
+            CTL_INF(res.error();)
+          }
+        }
       }
-    }
 
-    // Sync cortex.db with the upstream data
-    // add new model if it does not exist
-    // update model if it does exist
-    // delete model if remote has removed model
-    for (auto const& c : update_cand) {
-      AddModelSource(c);
+      CTL_DBG("Done sync cortex.db");
     }
   }
 }
diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h
index ebebe78b4..6737dee57 100644
--- a/engine/services/model_source_service.h
+++ b/engine/services/model_source_service.h
@@ -1,20 +1,15 @@
 #pragma once
+#include <thread>
+#include <unordered_set>
+#include <thread>
+#include <atomic>
 #include "utils/result.hpp"
 
-// struct ModelEntry {
-//   std::string model;
-//   std::string author_repo_id;
-//   std::string branch_name;
-//   std::string path_to_model_yaml;
-//   std::string model_alias;
-//   std::string model_format;
-//   std::string model_source;
-//   ModelStatus status;
-//   std::string engine;
-// };
 namespace services {
 class ModelSourceService {
  public:
+  explicit ModelSourceService();
+  ~ModelSourceService();
   // model source can be HF organization, repo or others (for example Modelscope,..)
   // default is HF, need to check if it is organization or repo
   // if repo:
@@ -28,17 +23,20 @@ class ModelSourceService {
   cpp::result<std::vector<std::string>, std::string> GetModelSources();
 
  private:
-  // models database
-  cpp::result<bool, std::string> AddOrg(const std::string& org);
-  cpp::result<bool, std::string> AddRepo(const std::string& model_source,
-                                         const std::string& author,
-                                         const std::string& model_name);
-  cpp::result<bool, std::string> AddCortexsoRepoBranch(
+  cpp::result<std::unordered_set<std::string>, std::string> AddRepo(
       const std::string& model_source, const std::string& author,
-      const std::string& model_name, const std::string& branch);
+      const std::string& model_name);
+
+  cpp::result<std::unordered_set<std::string>, std::string>
+  AddCortexsoRepoBranch(const std::string& model_source,
+                        const std::string& author,
+                        const std::string& model_name,
+                        const std::string& branch);
 
-  cpp::result<bool, std::string> RemoveOrg(const std::string& org);
-  cpp::result<bool, std::string> RemoveRepo(const std::string& repo);
   void SyncModelSource();
+
+ private:
+ std::thread sync_db_thread_;
+ std::atomic<bool> running_;
 };
 }  // namespace services
\ No newline at end of file

From 875e4883931631a32ee0155dd00077ca8992d963 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Mon, 9 Dec 2024 10:54:20 +0700
Subject: [PATCH 12/21] chore: cleanup

---
 engine/cli/command_line_parser.cc       |   2 +-
 engine/cli/commands/model_list_cmd.cc   |   4 +-
 engine/cli/commands/model_list_cmd.h    |   2 +-
 engine/services/model_source_service.cc | 277 +++++++++++++-----------
 engine/services/model_source_service.h  |  23 +-
 5 files changed, 175 insertions(+), 133 deletions(-)

diff --git a/engine/cli/command_line_parser.cc b/engine/cli/command_line_parser.cc
index 962a13538..624ccd3dd 100644
--- a/engine/cli/command_line_parser.cc
+++ b/engine/cli/command_line_parser.cc
@@ -256,7 +256,7 @@ void CommandLineParser::SetupModelCommands() {
                             "Display cpu mode");
   list_models_cmd->add_flag("--gpu_mode", cml_data_.display_gpu_mode,
                             "Display gpu mode");
-  list_models_cmd->add_flag("--remote", cml_data_.display_available_model,
+  list_models_cmd->add_flag("--available", cml_data_.display_available_model,
                             "Display available models to download");
   list_models_cmd->group(kSubcommands);
   list_models_cmd->callback([this]() {
diff --git a/engine/cli/commands/model_list_cmd.cc b/engine/cli/commands/model_list_cmd.cc
index f33667f1f..03275f7fd 100644
--- a/engine/cli/commands/model_list_cmd.cc
+++ b/engine/cli/commands/model_list_cmd.cc
@@ -21,7 +21,7 @@ using Row_t =
 void ModelListCmd::Exec(const std::string& host, int port,
                         const std::string& filter, bool display_engine,
                         bool display_version, bool display_cpu_mode,
-                        bool display_gpu_mode, bool is_remote) {
+                        bool display_gpu_mode, bool available) {
   // Start server if server is not started yet
   if (!commands::IsServerAlive(host, port)) {
     CLI_LOG("Starting server ...");
@@ -73,7 +73,7 @@ void ModelListCmd::Exec(const std::string& host, int port,
         continue;
       }
 
-      if (is_remote) {
+      if (available) {
         if (v["status"].asString() != "undownloaded") {
           continue;
         }
diff --git a/engine/cli/commands/model_list_cmd.h b/engine/cli/commands/model_list_cmd.h
index 839888803..85dd76de9 100644
--- a/engine/cli/commands/model_list_cmd.h
+++ b/engine/cli/commands/model_list_cmd.h
@@ -9,6 +9,6 @@ class ModelListCmd {
   void Exec(const std::string& host, int port, const std::string& filter,
             bool display_engine = false, bool display_version = false,
             bool display_cpu_mode = false, bool display_gpu_mode = false,
-            bool is_remote = false);
+            bool available = false);
 };
 }  // namespace commands
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index 6bfaa7a06..e856570ba 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -74,8 +74,6 @@ ModelSourceService::~ModelSourceService() {
 
 cpp::result<bool, std::string> ModelSourceService::AddModelSource(
     const std::string& model_source) {
-  // https://huggingface.co/Orenguteng
-  // https://huggingface.co/Orenguteng/Llama-3.1-8B-Lexi-Uncensored-V2-GGUF
   auto res = url_parser::FromUrlString(model_source);
   if (res.has_error()) {
     return cpp::fail(res.error());
@@ -88,131 +86,17 @@ cpp::result<bool, std::string> ModelSourceService::AddModelSource(
     if (auto is_org = r.pathParams.size() == 1; is_org) {
       auto& author = r.pathParams[0];
       if (author == "cortexso") {
-        if (auto res = curl_utils::SimpleGet(
-                "https://huggingface.co/api/models?author=" + author);
-            res.has_value()) {
-          auto models = ParseJsonString(res.value());
-          // Get models from db
-          cortex::db::Models model_db;
-
-          auto model_list_before = model_db.GetModels(model_source)
-                                       .value_or(std::vector<std::string>{});
-          std::unordered_set<std::string> updated_model_list;
-          for (auto const& m : models) {
-            CTL_INF(m.id);
-            auto author_model = string_utils::SplitBy(m.id, "/");
-            if (author_model.size() == 2) {
-              auto const& author = author_model[0];
-              auto const& model_name = author_model[1];
-              auto branches = huggingface_utils::GetModelRepositoryBranches(
-                  "cortexso", model_name);
-              if (branches.has_error()) {
-                CTL_INF(branches.error());
-                continue;
-              }
-              for (auto const& [branch, _] : branches.value()) {
-                CTL_INF(branch);
-                auto add_res = AddCortexsoRepoBranch(model_source, author,
-                                                     model_name, branch)
-                                   .value_or(std::unordered_set<std::string>{});
-                for (auto const& a : add_res) {
-                  updated_model_list.insert(a);
-                }
-              }
-            }
-          }
-          // Clean up
-          for (auto const& mid : model_list_before) {
-            if (updated_model_list.find(mid) == updated_model_list.end()) {
-              model_db.DeleteModelEntry(mid);
-            }
-          }
-        }
+        return AddCortexsoOrg(model_source);
       } else {
-        if (auto res = curl_utils::SimpleGet(
-                "https://huggingface.co/api/models?author=" + author);
-            res.has_value()) {
-          auto models = ParseJsonString(res.value());
-          // Get models from db
-          cortex::db::Models model_db;
-
-          auto model_list_before = model_db.GetModels(model_source)
-                                       .value_or(std::vector<std::string>{});
-          std::unordered_set<std::string> updated_model_list;
-          // Add new models
-          for (auto const& m : models) {
-            CTL_INF(m.id);
-            auto author_model = string_utils::SplitBy(m.id, "/");
-            if (author_model.size() == 2) {
-              auto const& author = author_model[0];
-              auto const& model_name = author_model[1];
-              auto add_res = AddRepo(model_source, author, model_name)
-                                 .value_or(std::unordered_set<std::string>{});
-              for (auto const& a : add_res) {
-                updated_model_list.insert(a);
-              }
-            }
-          }
-
-          // Clean up
-          for (auto const& mid : model_list_before) {
-            if (updated_model_list.find(mid) == updated_model_list.end()) {
-              model_db.DeleteModelEntry(mid);
-            }
-          }
-        }
+        return AddHfOrg(model_source, author);
       }
-
     } else {  // Repo
       auto const& author = r.pathParams[0];
       auto const& model_name = r.pathParams[1];
       if (r.pathParams[0] == "cortexso") {
-        auto branches = huggingface_utils::GetModelRepositoryBranches(
-            "cortexso", model_name);
-        if (branches.has_error()) {
-          return cpp::fail(branches.error());
-        }
-        // Get models from db
-        cortex::db::Models model_db;
-
-        auto model_list_before = model_db.GetModels(model_source)
-                                     .value_or(std::vector<std::string>{});
-        std::unordered_set<std::string> updated_model_list;
-
-        for (auto const& [branch, _] : branches.value()) {
-          CTL_INF(branch);
-          auto add_res =
-              AddCortexsoRepoBranch(model_source, author, model_name, branch)
-                  .value_or(std::unordered_set<std::string>{});
-          for (auto const& a : add_res) {
-            updated_model_list.insert(a);
-          }
-        }
-
-        // Clean up
-        for (auto const& mid : model_list_before) {
-          if (updated_model_list.find(mid) == updated_model_list.end()) {
-            model_db.DeleteModelEntry(mid);
-          }
-        }
+        return AddCortexsoRepo(model_source, author, model_name);
       } else {
-        // Get models from db
-        cortex::db::Models model_db;
-
-        auto model_list_before = model_db.GetModels(model_source)
-                                     .value_or(std::vector<std::string>{});
-        std::unordered_set<std::string> updated_model_list;
-        auto add_res = AddRepo(model_source, author, model_name);
-        if (res.has_error()) {
-          return cpp::fail(res.error());
-        } else {
-          updated_model_list = add_res.value();
-        }
-        for (auto const& mid : model_list_before) {
-          if (updated_model_list.find(mid) == updated_model_list.end()) {
-            model_db.DeleteModelEntry(mid);
-          }
-        }
+        return AddHfRepo(model_source, author, model_name);
       }
     }
   }
@@ -264,8 +148,70 @@ ModelSourceService::GetModelSources() {
   return model_db.GetModelSources();
 }
 
+cpp::result<bool, std::string> ModelSourceService::AddHfOrg(
+    const std::string& model_source, const std::string& author) {
+  auto res = curl_utils::SimpleGet("https://huggingface.co/api/models?author=" +
+                                   author);
+  if (res.has_value()) {
+    auto models = ParseJsonString(res.value());
+    // Get models from db
+    cortex::db::Models model_db;
+
+    auto model_list_before =
+        model_db.GetModels(model_source).value_or(std::vector<std::string>{});
+    std::unordered_set<std::string> updated_model_list;
+    // Add new models
+    for (auto const& m : models) {
+      CTL_INF(m.id);
+      auto author_model = string_utils::SplitBy(m.id, "/");
+      if (author_model.size() == 2) {
+        auto const& author = author_model[0];
+        auto const& model_name = author_model[1];
+        auto add_res = AddRepoSiblings(model_source, author, model_name)
+                           .value_or(std::unordered_set<std::string>{});
+        for (auto const& a : add_res) {
+          updated_model_list.insert(a);
+        }
+      }
+    }
+
+    // Clean up
+    for (auto const& mid : model_list_before) {
+      if (updated_model_list.find(mid) == updated_model_list.end()) {
+        (void) model_db.DeleteModelEntry(mid);
+      }
+    }
+  } else {
+    return cpp::fail(res.error());
+  }
+  return true;
+}
+
+cpp::result<bool, std::string> ModelSourceService::AddHfRepo(
+    const std::string& model_source, const std::string& author,
+    const std::string& model_name) {
+  // Get models from db
+  cortex::db::Models model_db;
+
+  auto model_list_before =
+      model_db.GetModels(model_source).value_or(std::vector<std::string>{});
+  std::unordered_set<std::string> updated_model_list;
+  auto add_res = AddRepoSiblings(model_source, author, model_name);
+  if (add_res.has_error()) {
+    return cpp::fail(add_res.error());
+  } else {
+    updated_model_list = add_res.value();
+  }
+  for (auto const& mid : model_list_before) {
+    if (updated_model_list.find(mid) == updated_model_list.end()) {
+      (void) model_db.DeleteModelEntry(mid);
+    }
+  }
+  return true;
+}
+
 cpp::result<std::unordered_set<std::string>, std::string>
-ModelSourceService::AddRepo(const std::string& model_source,
+ModelSourceService::AddRepoSiblings(const std::string& model_source,
                             const std::string& author,
                             const std::string& model_name) {
   std::unordered_set<std::string> res;
@@ -296,9 +242,9 @@ ModelSourceService::AddRepo(const std::string& model_source,
             .model_source = model_source,
             .status = cortex::db::ModelStatus::Undownloaded,
             .engine = "llama-cpp"};
-        model_db.AddModelEntry(e);
+        (void) model_db.AddModelEntry(e);
       } else {
-        // update
+        // update metadata
       }
       res.insert(model_id);
     }
@@ -307,6 +253,88 @@ ModelSourceService::AddRepo(const std::string& model_source,
   return res;
 }
 
+cpp::result<bool, std::string> ModelSourceService::AddCortexsoOrg(
+    const std::string& model_source) {
+  auto res = curl_utils::SimpleGet(
+      "https://huggingface.co/api/models?author=cortexso");
+  if (res.has_value()) {
+    auto models = ParseJsonString(res.value());
+    // Get models from db
+    cortex::db::Models model_db;
+
+    auto model_list_before =
+        model_db.GetModels(model_source).value_or(std::vector<std::string>{});
+    std::unordered_set<std::string> updated_model_list;
+    for (auto const& m : models) {
+      CTL_INF(m.id);
+      auto author_model = string_utils::SplitBy(m.id, "/");
+      if (author_model.size() == 2) {
+        auto const& author = author_model[0];
+        auto const& model_name = author_model[1];
+        auto branches = huggingface_utils::GetModelRepositoryBranches(
+            "cortexso", model_name);
+        if (branches.has_error()) {
+          CTL_INF(branches.error());
+          continue;
+        }
+        for (auto const& [branch, _] : branches.value()) {
+          CTL_INF(branch);
+          auto add_res =
+              AddCortexsoRepoBranch(model_source, author, model_name, branch)
+                  .value_or(std::unordered_set<std::string>{});
+          for (auto const& a : add_res) {
+            updated_model_list.insert(a);
+          }
+        }
+      }
+    }
+    // Clean up
+    for (auto const& mid : model_list_before) {
+      if (updated_model_list.find(mid) == updated_model_list.end()) {
+        (void) model_db.DeleteModelEntry(mid);
+      }
+    }
+  } else {
+    return cpp::fail(res.error());
+  }
+
+  return true;
+}
+
+cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepo(
+    const std::string& model_source, const std::string& author,
+    const std::string& model_name) {
+  auto branches =
+      huggingface_utils::GetModelRepositoryBranches("cortexso", model_name);
+  if (branches.has_error()) {
+    return cpp::fail(branches.error());
+  }
+  // Get models from db
+  cortex::db::Models model_db;
+
+  auto model_list_before =
+      model_db.GetModels(model_source).value_or(std::vector<std::string>{});
+  std::unordered_set<std::string> updated_model_list;
+
+  for (auto const& [branch, _] : branches.value()) {
+    CTL_INF(branch);
+    auto add_res =
+        AddCortexsoRepoBranch(model_source, author, model_name, branch)
+            .value_or(std::unordered_set<std::string>{});
+    for (auto const& a : add_res) {
+      updated_model_list.insert(a);
+    }
+  }
+
+  // Clean up
+  for (auto const& mid : model_list_before) {
+    if (updated_model_list.find(mid) == updated_model_list.end()) {
+      (void) model_db.DeleteModelEntry(mid);
+    }
+  }
+  return true;
+}
+
 cpp::result<std::unordered_set<std::string>, std::string>
 ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source,
                                           const std::string& author,
@@ -355,6 +383,7 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source,
         CTL_DBG("Cannot add model to db: " << model_id);
       }
     } else {
+      // Update metadata?
       CTL_DBG("Model exists: " << model_id);
     }
     res.insert(model_id);
@@ -412,7 +441,7 @@ void ModelSourceService::SyncModelSource() {
 
         // Sync cortex.db with the upstream data
         for (auto const& c : update_cand) {
-          if(auto res = AddModelSource(c); res.has_error()) {
+          if (auto res = AddModelSource(c); res.has_error()) {
             CTL_INF(res.error();)
           }
         }
diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h
index 6737dee57..c668720b6 100644
--- a/engine/services/model_source_service.h
+++ b/engine/services/model_source_service.h
@@ -1,8 +1,7 @@
 #pragma once
+#include <atomic>
 #include <thread>
 #include <unordered_set>
-#include <thread>
-#include <atomic>
 #include "utils/result.hpp"
 
 namespace services {
@@ -23,7 +22,21 @@ class ModelSourceService {
   cpp::result<std::vector<std::string>, std::string> GetModelSources();
 
  private:
-  cpp::result<std::unordered_set<std::string>, std::string> AddRepo(
+  cpp::result<bool, std::string> AddHfOrg(const std::string& model_source,
+                                          const std::string& author);
+
+  cpp::result<bool, std::string> AddHfRepo(
+      const std::string& model_source, const std::string& author,
+      const std::string& model_name);
+
+  cpp::result<std::unordered_set<std::string>, std::string> AddRepoSiblings(
+      const std::string& model_source, const std::string& author,
+      const std::string& model_name);
+
+  cpp::result<bool, std::string> AddCortexsoOrg(
+      const std::string& model_source);
+
+  cpp::result<bool, std::string> AddCortexsoRepo(
       const std::string& model_source, const std::string& author,
       const std::string& model_name);
 
@@ -36,7 +49,7 @@ class ModelSourceService {
   void SyncModelSource();
 
  private:
- std::thread sync_db_thread_;
- std::atomic<bool> running_;
+  std::thread sync_db_thread_;
+  std::atomic<bool> running_;
 };
 }  // namespace services
\ No newline at end of file

From afafe33c1f0aaf1689bab13dd48f4c6670e19125 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Mon, 9 Dec 2024 11:16:03 +0700
Subject: [PATCH 13/21] feat: add metadata for model

---
 engine/controllers/models.cc            |  1 +
 engine/database/models.h                |  1 +
 engine/services/model_source_service.cc | 38 +++++++++++++++++++------
 engine/services/model_source_service.h  |  5 +---
 4 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc
index 83adc6352..961c16602 100644
--- a/engine/controllers/models.cc
+++ b/engine/controllers/models.cc
@@ -190,6 +190,7 @@ void Models::ListModel(
           obj["modelSource"] = model_entry.model_source;
           obj["status"] = status_to_string(model_entry.status);
           obj["engine"] = model_entry.engine;
+          obj["metadata"] = model_entry.metadata;
           data.append(std::move(obj));
           continue;
         }
diff --git a/engine/database/models.h b/engine/database/models.h
index 79d6d878d..c0f308741 100644
--- a/engine/database/models.h
+++ b/engine/database/models.h
@@ -20,6 +20,7 @@ struct ModelEntry {
   std::string model_source;
   ModelStatus status;
   std::string engine;
+  std::string metadata;
 };
 
 class Models {
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index e856570ba..d3ebb339e 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -2,6 +2,7 @@
 #include <chrono>
 #include <unordered_set>
 #include "database/models.h"
+#include "json/json.h"
 #include "utils/curl_utils.h"
 #include "utils/huggingface_utils.h"
 #include "utils/logging_utils.h"
@@ -21,6 +22,27 @@ struct ModelInfo {
   std::vector<std::string> tags;
   std::string created_at;
   std::string model_id;
+
+  std::string ToJsonString() {
+    Json::Value root;
+    root["id"] = id;
+    root["likes"] = likes;
+    root["trendingScore"] = trending_score;
+    root["isPrivate"] = is_private;
+    root["downloads"] = downloads;
+
+    Json::Value tags_array(Json::arrayValue);
+    for (const auto& tag : tags) {
+      tags_array.append(tag);
+    }
+    root["tags"] = tags_array;
+
+    root["createdAt"] = created_at;
+    root["modelId"] = model_id;
+
+    Json::StreamWriterBuilder builder;
+    return Json::writeString(builder, root);
+  }
 };
 
 std::vector<ModelInfo> ParseJsonString(const std::string& json_str) {
@@ -178,7 +200,7 @@ cpp::result<bool, std::string> ModelSourceService::AddHfOrg(
     // Clean up
     for (auto const& mid : model_list_before) {
       if (updated_model_list.find(mid) == updated_model_list.end()) {
-        (void) model_db.DeleteModelEntry(mid);
+        (void)model_db.DeleteModelEntry(mid);
       }
     }
   } else {
@@ -204,7 +226,7 @@ cpp::result<bool, std::string> ModelSourceService::AddHfRepo(
   }
   for (auto const& mid : model_list_before) {
     if (updated_model_list.find(mid) == updated_model_list.end()) {
-      (void) model_db.DeleteModelEntry(mid);
+      (void)model_db.DeleteModelEntry(mid);
     }
   }
   return true;
@@ -212,8 +234,8 @@ cpp::result<bool, std::string> ModelSourceService::AddHfRepo(
 
 cpp::result<std::unordered_set<std::string>, std::string>
 ModelSourceService::AddRepoSiblings(const std::string& model_source,
-                            const std::string& author,
-                            const std::string& model_name) {
+                                    const std::string& author,
+                                    const std::string& model_name) {
   std::unordered_set<std::string> res;
   auto repo_info = hu::GetHuggingFaceModelRepoInfo(author, model_name);
   if (repo_info.has_error()) {
@@ -242,9 +264,9 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source,
             .model_source = model_source,
             .status = cortex::db::ModelStatus::Undownloaded,
             .engine = "llama-cpp"};
-        (void) model_db.AddModelEntry(e);
+        (void)model_db.AddModelEntry(e);
       } else {
-        // update metadata
+        // TODO(sang) update metadata
       }
       res.insert(model_id);
     }
@@ -291,7 +313,7 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoOrg(
     // Clean up
     for (auto const& mid : model_list_before) {
       if (updated_model_list.find(mid) == updated_model_list.end()) {
-        (void) model_db.DeleteModelEntry(mid);
+        (void)model_db.DeleteModelEntry(mid);
       }
     }
   } else {
@@ -329,7 +351,7 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepo(
   // Clean up
   for (auto const& mid : model_list_before) {
     if (updated_model_list.find(mid) == updated_model_list.end()) {
-      (void) model_db.DeleteModelEntry(mid);
+      (void)model_db.DeleteModelEntry(mid);
     }
   }
   return true;
diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h
index c668720b6..b84d2720f 100644
--- a/engine/services/model_source_service.h
+++ b/engine/services/model_source_service.h
@@ -9,10 +9,7 @@ class ModelSourceService {
  public:
   explicit ModelSourceService();
   ~ModelSourceService();
-  // model source can be HF organization, repo or others (for example Modelscope,..)
-  // default is HF, need to check if it is organization or repo
-  // if repo:
-  // if org: api/models?author=cortexso
+  
   cpp::result<bool, std::string> AddModelSource(
       const std::string& model_source);
 

From f9f600248fc761c6ba5bdedbde5de72205c44b76 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Mon, 9 Dec 2024 11:26:47 +0700
Subject: [PATCH 14/21] fix: migration

---
 engine/migrations/migration_manager.cc |  4 ----
 engine/migrations/v2/migration.h       | 20 ++++++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc
index 730ecb856..6936f45a0 100644
--- a/engine/migrations/migration_manager.cc
+++ b/engine/migrations/migration_manager.cc
@@ -141,7 +141,6 @@ cpp::result<bool, std::string> MigrationManager::DoUpFolderStructure(
   switch (version) {
     case 0:
       return v0::MigrateFolderStructureUp();
-      break;
     case 1:
       return v1::MigrateFolderStructureUp();
     case 2:
@@ -158,7 +157,6 @@ cpp::result<bool, std::string> MigrationManager::DoDownFolderStructure(
   switch (version) {
     case 0:
       return v0::MigrateFolderStructureDown();
-      break;
     case 1:
       return v1::MigrateFolderStructureDown();
     case 2:
@@ -196,7 +194,6 @@ cpp::result<bool, std::string> MigrationManager::DoUpDB(int version) {
   switch (version) {
     case 0:
       return v0::MigrateDBUp(db_);
-      break;
     case 1:
       return v1::MigrateDBUp(db_);
     case 2:
@@ -212,7 +209,6 @@ cpp::result<bool, std::string> MigrationManager::DoDownDB(int version) {
   switch (version) {
     case 0:
       return v0::MigrateDBDown(db_);
-      break;
     case 1:
       return v1::MigrateDBDown(db_);
     case 2:
diff --git a/engine/migrations/v2/migration.h b/engine/migrations/v2/migration.h
index d28020645..54b79f666 100644
--- a/engine/migrations/v2/migration.h
+++ b/engine/migrations/v2/migration.h
@@ -60,10 +60,7 @@ inline cpp::result<bool, std::string> MigrateDBUp(SQLite::Database& db) {
 
       if (table_exists) {
         // Alter existing table
-        cortex::mgr::AddColumnIfNotExists(db, "models", "model_format", "TEXT");
-        cortex::mgr::AddColumnIfNotExists(db, "models", "model_source", "TEXT");
-        cortex::mgr::AddColumnIfNotExists(db, "models", "status", "TEXT");
-        cortex::mgr::AddColumnIfNotExists(db, "models", "engine", "TEXT");
+        cortex::mgr::AddColumnIfNotExists(db, "models", "metadata", "TEXT");
       } else {
         // Create new table
         db.exec(
@@ -76,7 +73,8 @@ inline cpp::result<bool, std::string> MigrateDBUp(SQLite::Database& db) {
             "model_format TEXT,"
             "model_source TEXT,"
             "status TEXT,"
-            "engine TEXT"
+            "engine TEXT,"
+            "metadata TEXT"
             ")");
       }
     }
@@ -141,15 +139,21 @@ inline cpp::result<bool, std::string> MigrateDBDown(SQLite::Database& db) {
             "author_repo_id TEXT,"
             "branch_name TEXT,"
             "path_to_model_yaml TEXT,"
-            "model_alias TEXT"
+            "model_alias TEXT,"
+            "model_format TEXT,"
+            "model_source TEXT,"
+            "status TEXT,"
+            "engine TEXT"
             ")");
 
         // Copy data from the current table to the new table
         db.exec(
             "INSERT INTO models_old (model_id, author_repo_id, branch_name, "
-            "path_to_model_yaml, model_alias) "
+            "path_to_model_yaml, model_alias, model_format, model_source, "
+            "status, engine) "
             "SELECT model_id, author_repo_id, branch_name, path_to_model_yaml, "
-            "model_alias FROM models");
+            "model_alias, model_format, model_source, status, engine FROM "
+            "models");
 
         // Drop the current table
         db.exec("DROP TABLE models");

From 1fcb68959953ef0cffe25e4d527fe074854ab3fd Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Mon, 9 Dec 2024 11:32:32 +0700
Subject: [PATCH 15/21] chore: unit tests: cleanup

---
 engine/test/components/test_models_db.cc | 42 ------------------------
 1 file changed, 42 deletions(-)

diff --git a/engine/test/components/test_models_db.cc b/engine/test/components/test_models_db.cc
index bce72a084..decf8b769 100644
--- a/engine/test/components/test_models_db.cc
+++ b/engine/test/components/test_models_db.cc
@@ -116,48 +116,6 @@ TEST_F(ModelsTestSuite, TestPersistence) {
   EXPECT_TRUE(model_list_.DeleteModelEntry(kTestModel.model).value());
 }
 
-TEST_F(ModelsTestSuite, TestUpdateModelAlias) {
-  constexpr const auto kNewTestAlias = "new_test_alias";
-  constexpr const auto kNonExistentModel = "non_existent_model";
-  constexpr const auto kAnotherAlias = "another_alias";
-  constexpr const auto kFinalTestAlias = "final_test_alias";
-  constexpr const auto kAnotherModelId = "another_model_id";
-  // Add the test model
-  ASSERT_TRUE(model_list_.AddModelEntry(kTestModel).value());
-
-  // Test successful update
-  EXPECT_TRUE(
-      model_list_.UpdateModelAlias(kTestModel.model, kNewTestAlias).value());
-  auto updated_model = model_list_.GetModelInfo(kNewTestAlias);
-  EXPECT_TRUE(updated_model.has_value());
-  EXPECT_EQ(updated_model.value().model_alias, kNewTestAlias);
-  EXPECT_EQ(updated_model.value().model, kTestModel.model);
-
-  // Test update with non-existent model
-  EXPECT_TRUE(model_list_.UpdateModelAlias(kNonExistentModel, kAnotherAlias)
-                  .has_error());
-
-  // Test update with non-unique alias
-  cortex::db::ModelEntry another_model = kTestModel;
-  another_model.model = kAnotherModelId;
-  another_model.model_alias = kAnotherAlias;
-  ASSERT_TRUE(model_list_.AddModelEntry(another_model).value());
-
-  EXPECT_FALSE(
-      model_list_.UpdateModelAlias(kTestModel.model, kAnotherAlias).value());
-
-  // Test update using model alias instead of model ID
-  EXPECT_TRUE(model_list_.UpdateModelAlias(kNewTestAlias, kFinalTestAlias));
-  updated_model = model_list_.GetModelInfo(kFinalTestAlias);
-  EXPECT_TRUE(updated_model);
-  EXPECT_EQ(updated_model.value().model_alias, kFinalTestAlias);
-  EXPECT_EQ(updated_model.value().model, kTestModel.model);
-
-  // Clean up
-  EXPECT_TRUE(model_list_.DeleteModelEntry(kTestModel.model).value());
-  EXPECT_TRUE(model_list_.DeleteModelEntry(kAnotherModelId).value());
-}
-
 TEST_F(ModelsTestSuite, TestHasModel) {
   EXPECT_TRUE(model_list_.AddModelEntry(kTestModel).value());
 

From 3707a181b70dc289969d4fc4d988e82db8627212 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Mon, 9 Dec 2024 13:02:17 +0700
Subject: [PATCH 16/21] fix: add metadata

---
 engine/database/models.cc               | 53 ++++----------
 engine/database/models.h                |  2 -
 engine/services/model_source_service.cc | 96 ++++++++++++-------------
 engine/services/model_source_service.h  |  3 +-
 engine/utils/huggingface_utils.h        |  2 +
 5 files changed, 60 insertions(+), 96 deletions(-)

diff --git a/engine/database/models.cc b/engine/database/models.cc
index f0a38c562..c4365eb68 100644
--- a/engine/database/models.cc
+++ b/engine/database/models.cc
@@ -63,7 +63,7 @@ cpp::result<std::vector<ModelEntry>, std::string> Models::LoadModelListNoLock()
     SQLite::Statement query(db_,
                             "SELECT model_id, author_repo_id, branch_name, "
                             "path_to_model_yaml, model_alias, model_format, "
-                            "model_source, status, engine FROM models");
+                            "model_source, status, engine, metadata FROM models");
 
     while (query.executeStep()) {
       ModelEntry entry;
@@ -76,6 +76,7 @@ cpp::result<std::vector<ModelEntry>, std::string> Models::LoadModelListNoLock()
       entry.model_source = query.getColumn(6).getString();
       entry.status = StringToStatus(query.getColumn(7).getString());
       entry.engine = query.getColumn(8).getString();
+      entry.metadata = query.getColumn(9).getString();
       entries.push_back(entry);
     }
     return entries;
@@ -91,11 +92,10 @@ cpp::result<ModelEntry, std::string> Models::GetModelInfo(
     SQLite::Statement query(db_,
                             "SELECT model_id, author_repo_id, branch_name, "
                             "path_to_model_yaml, model_alias, model_format, "
-                            "model_source, status, engine FROM models "
-                            "WHERE model_id = ? OR model_alias = ?");
+                            "model_source, status, engine, metadata FROM models "
+                            "WHERE model_id = ?");
 
     query.bind(1, identifier);
-    query.bind(2, identifier);
     if (query.executeStep()) {
       ModelEntry entry;
       entry.model = query.getColumn(0).getString();
@@ -107,6 +107,7 @@ cpp::result<ModelEntry, std::string> Models::GetModelInfo(
       entry.model_source = query.getColumn(6).getString();
       entry.status = StringToStatus(query.getColumn(7).getString());
       entry.engine = query.getColumn(8).getString();
+      entry.metadata = query.getColumn(9).getString();
       return entry;
     } else {
       return cpp::fail("Model not found: " + identifier);
@@ -126,6 +127,7 @@ void Models::PrintModelInfo(const ModelEntry& entry) const {
   LOG_INFO << "Model Source: " << entry.model_source;
   LOG_INFO << "Status: " << StatusToString(entry.status);
   LOG_INFO << "Engine: " << entry.engine;
+  LOG_INFO << "Metadata: " << entry.metadata;
 }
 
 cpp::result<bool, std::string> Models::AddModelEntry(ModelEntry new_entry) {
@@ -143,7 +145,7 @@ cpp::result<bool, std::string> Models::AddModelEntry(ModelEntry new_entry) {
           db_,
           "INSERT INTO models (model_id, author_repo_id, branch_name, "
           "path_to_model_yaml, model_alias, model_format, model_source, "
-          "status, engine) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)");
+          "status, engine, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
       insert.bind(1, new_entry.model);
       insert.bind(2, new_entry.author_repo_id);
       insert.bind(3, new_entry.branch_name);
@@ -153,6 +155,7 @@ cpp::result<bool, std::string> Models::AddModelEntry(ModelEntry new_entry) {
       insert.bind(7, new_entry.model_source);
       insert.bind(8, StatusToString(new_entry.status));
       insert.bind(9, new_entry.engine);
+      insert.bind(10, new_entry.metadata);
       insert.exec();
 
       return true;
@@ -174,7 +177,7 @@ cpp::result<bool, std::string> Models::UpdateModelEntry(
         db_,
         "UPDATE models SET author_repo_id = ?, branch_name = ?, "
         "path_to_model_yaml = ?, model_format = ?, model_source = ?, status = "
-        "?, engine = ? WHERE model_id = ? OR model_alias = ?");
+        "?, engine = ?, metadata = ? WHERE model_id = ?");
     upd.bind(1, updated_entry.author_repo_id);
     upd.bind(2, updated_entry.branch_name);
     upd.bind(3, updated_entry.path_to_model_yaml);
@@ -182,7 +185,7 @@ cpp::result<bool, std::string> Models::UpdateModelEntry(
     upd.bind(5, updated_entry.model_source);
     upd.bind(6, StatusToString(updated_entry.status));
     upd.bind(7, updated_entry.engine);
-    upd.bind(8, identifier);
+    upd.bind(8, updated_entry.metadata);
     upd.bind(9, identifier);
     return upd.exec() == 1;
   } catch (const std::exception& e) {
@@ -190,36 +193,6 @@ cpp::result<bool, std::string> Models::UpdateModelEntry(
   }
 }
 
-cpp::result<bool, std::string> Models::UpdateModelAlias(
-    const std::string& model_id, const std::string& new_model_alias) {
-  if (!HasModel(model_id)) {
-    return cpp::fail("Model not found: " + model_id);
-  }
-  try {
-    db_.exec("BEGIN TRANSACTION;");
-    cortex::utils::ScopeExit se([this] { db_.exec("COMMIT;"); });
-    auto model_list = LoadModelListNoLock();
-    if (model_list.has_error()) {
-      CTL_WRN(model_list.error());
-      return cpp::fail(model_list.error());
-    }
-    // Check new_model_alias is unique
-    if (IsUnique(model_list.value(), new_model_alias)) {
-      SQLite::Statement upd(db_,
-                            "UPDATE models "
-                            "SET model_alias = ? "
-                            "WHERE model_id = ? OR model_alias = ?");
-      upd.bind(1, new_model_alias);
-      upd.bind(2, model_id);
-      upd.bind(3, model_id);
-      return upd.exec() == 1;
-    }
-    return false;
-  } catch (const std::exception& e) {
-    return cpp::fail(e.what());
-  }
-}
-
 cpp::result<bool, std::string> Models::DeleteModelEntry(
     const std::string& identifier) {
   try {
@@ -229,9 +202,8 @@ cpp::result<bool, std::string> Models::DeleteModelEntry(
     }
 
     SQLite::Statement del(
-        db_, "DELETE from models WHERE model_id = ? OR model_alias = ?");
+        db_, "DELETE from models WHERE model_id = ?");
     del.bind(1, identifier);
-    del.bind(2, identifier);
     return del.exec() == 1;
   } catch (const std::exception& e) {
     return cpp::fail(e.what());
@@ -285,9 +257,8 @@ bool Models::HasModel(const std::string& identifier) const {
   try {
     SQLite::Statement query(
         db_,
-        "SELECT COUNT(*) FROM models WHERE model_id = ? OR model_alias = ?");
+        "SELECT COUNT(*) FROM models WHERE model_id = ?");
     query.bind(1, identifier);
-    query.bind(2, identifier);
     if (query.executeStep()) {
       return query.getColumn(0).getInt() > 0;
     }
diff --git a/engine/database/models.h b/engine/database/models.h
index c0f308741..034acd825 100644
--- a/engine/database/models.h
+++ b/engine/database/models.h
@@ -53,8 +53,6 @@ class Models {
       const std::string& src);
   cpp::result<bool, std::string> DeleteModelEntryWithRepo(
       const std::string& src);
-  cpp::result<bool, std::string> UpdateModelAlias(
-      const std::string& model_id, const std::string& model_alias);
   cpp::result<std::vector<std::string>, std::string> FindRelatedModel(
       const std::string& identifier) const;
   bool HasModel(const std::string& identifier) const;
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index d3ebb339e..c40e2b724 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -22,27 +22,6 @@ struct ModelInfo {
   std::vector<std::string> tags;
   std::string created_at;
   std::string model_id;
-
-  std::string ToJsonString() {
-    Json::Value root;
-    root["id"] = id;
-    root["likes"] = likes;
-    root["trendingScore"] = trending_score;
-    root["isPrivate"] = is_private;
-    root["downloads"] = downloads;
-
-    Json::Value tags_array(Json::arrayValue);
-    for (const auto& tag : tags) {
-      tags_array.append(tag);
-    }
-    root["tags"] = tags_array;
-
-    root["createdAt"] = created_at;
-    root["modelId"] = model_id;
-
-    Json::StreamWriterBuilder builder;
-    return Json::writeString(builder, root);
-  }
 };
 
 std::vector<ModelInfo> ParseJsonString(const std::string& json_str) {
@@ -184,7 +163,7 @@ cpp::result<bool, std::string> ModelSourceService::AddHfOrg(
     std::unordered_set<std::string> updated_model_list;
     // Add new models
     for (auto const& m : models) {
-      CTL_INF(m.id);
+      CTL_DBG(m.id);
       auto author_model = string_utils::SplitBy(m.id, "/");
       if (author_model.size() == 2) {
         auto const& author = author_model[0];
@@ -253,20 +232,21 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source,
       cortex::db::Models model_db;
       std::string model_id =
           author + ":" + model_name + ":" + sibling.rfilename;
+      cortex::db::ModelEntry e = {
+          .model = model_id,
+          .author_repo_id = author,
+          .branch_name = "main",
+          .path_to_model_yaml = "",
+          .model_alias = "",
+          .model_format = "hf-gguf",
+          .model_source = model_source,
+          .status = cortex::db::ModelStatus::Undownloaded,
+          .engine = "llama-cpp",
+          .metadata = repo_info->metadata};
       if (!model_db.HasModel(model_id)) {
-        cortex::db::ModelEntry e = {
-            .model = model_id,
-            .author_repo_id = author,
-            .branch_name = "main",
-            .path_to_model_yaml = "",
-            .model_alias = "",
-            .model_format = "hf-gguf",
-            .model_source = model_source,
-            .status = cortex::db::ModelStatus::Undownloaded,
-            .engine = "llama-cpp"};
         (void)model_db.AddModelEntry(e);
       } else {
-        // TODO(sang) update metadata
+        (void)model_db.UpdateModelEntry(model_id, e);
       }
       res.insert(model_id);
     }
@@ -299,11 +279,17 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoOrg(
           CTL_INF(branches.error());
           continue;
         }
+
+        auto repo_info = hu::GetHuggingFaceModelRepoInfo(author, model_name);
+        if (repo_info.has_error()) {
+          CTL_INF(repo_info.error());
+          continue;
+        }
         for (auto const& [branch, _] : branches.value()) {
           CTL_INF(branch);
-          auto add_res =
-              AddCortexsoRepoBranch(model_source, author, model_name, branch)
-                  .value_or(std::unordered_set<std::string>{});
+          auto add_res = AddCortexsoRepoBranch(model_source, author, model_name,
+                                               branch, repo_info->metadata)
+                             .value_or(std::unordered_set<std::string>{});
           for (auto const& a : add_res) {
             updated_model_list.insert(a);
           }
@@ -331,6 +317,11 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepo(
   if (branches.has_error()) {
     return cpp::fail(branches.error());
   }
+
+  auto repo_info = hu::GetHuggingFaceModelRepoInfo(author, model_name);
+  if (repo_info.has_error()) {
+    return cpp::fail(repo_info.error());
+  }
   // Get models from db
   cortex::db::Models model_db;
 
@@ -340,9 +331,9 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepo(
 
   for (auto const& [branch, _] : branches.value()) {
     CTL_INF(branch);
-    auto add_res =
-        AddCortexsoRepoBranch(model_source, author, model_name, branch)
-            .value_or(std::unordered_set<std::string>{});
+    auto add_res = AddCortexsoRepoBranch(model_source, author, model_name,
+                                         branch, repo_info->metadata)
+                       .value_or(std::unordered_set<std::string>{});
     for (auto const& a : add_res) {
       updated_model_list.insert(a);
     }
@@ -361,7 +352,8 @@ cpp::result<std::unordered_set<std::string>, std::string>
 ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source,
                                           const std::string& author,
                                           const std::string& model_name,
-                                          const std::string& branch) {
+                                          const std::string& branch,
+                                          const std::string& metadata) {
   std::unordered_set<std::string> res;
 
   url_parser::Url url = {
@@ -388,25 +380,25 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source,
   } else {
     cortex::db::Models model_db;
     std::string model_id = model_name + ":" + branch;
+    cortex::db::ModelEntry e = {.model = model_id,
+                                .author_repo_id = author,
+                                .branch_name = branch,
+                                .path_to_model_yaml = "",
+                                .model_alias = "",
+                                .model_format = "cortexso",
+                                .model_source = model_source,
+                                .status = cortex::db::ModelStatus::Undownloaded,
+                                .engine = "llama-cpp",
+                                .metadata = metadata};
     if (!model_db.HasModel(model_id)) {
       CTL_INF("Adding model to db: " << model_name << ":" << branch);
-      cortex::db::ModelEntry e = {
-          .model = model_id,
-          .author_repo_id = author,
-          .branch_name = branch,
-          .path_to_model_yaml = "",
-          .model_alias = "",
-          .model_format = "cortexso",
-          .model_source = model_source,
-          .status = cortex::db::ModelStatus::Undownloaded,
-          .engine = "llama-cpp"};
       if (auto res = model_db.AddModelEntry(e);
           res.has_error() || !res.value()) {
         CTL_DBG("Cannot add model to db: " << model_id);
       }
     } else {
-      // Update metadata?
-      CTL_DBG("Model exists: " << model_id);
+      (void)model_db.UpdateModelEntry(model_id, e);
+      CTL_DBG("Updated model: " << model_id);
     }
     res.insert(model_id);
   }
diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h
index b84d2720f..aa0b37259 100644
--- a/engine/services/model_source_service.h
+++ b/engine/services/model_source_service.h
@@ -41,7 +41,8 @@ class ModelSourceService {
   AddCortexsoRepoBranch(const std::string& model_source,
                         const std::string& author,
                         const std::string& model_name,
-                        const std::string& branch);
+                        const std::string& branch,
+                        const std::string& metadata);
 
   void SyncModelSource();
 
diff --git a/engine/utils/huggingface_utils.h b/engine/utils/huggingface_utils.h
index f2895c363..1d1040612 100644
--- a/engine/utils/huggingface_utils.h
+++ b/engine/utils/huggingface_utils.h
@@ -67,6 +67,7 @@ struct HuggingFaceModelRepoInfo {
   std::vector<HuggingFaceFileSibling> siblings;
   std::vector<std::string> spaces;
   std::string createdAt;
+  std::string metadata;
 
   static cpp::result<HuggingFaceModelRepoInfo, std::string> FromJson(
       const Json::Value& body) {
@@ -104,6 +105,7 @@ struct HuggingFaceModelRepoInfo {
         .spaces =
             json_parser_utils::ParseJsonArray<std::string>(body["spaces"]),
         .createdAt = body["createdAt"].asString(),
+        .metadata = body.toStyledString(),
     };
   }
 

From 5cda8f3d149691588bbfcc383ab28e179fee5d05 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Mon, 9 Dec 2024 14:00:47 +0700
Subject: [PATCH 17/21] fix: pull model

---
 engine/database/models.cc               |  33 ++++----
 engine/services/model_service.cc        | 106 +++++++++++++++++-------
 engine/services/model_source_service.cc |  40 +++++++--
 3 files changed, 126 insertions(+), 53 deletions(-)

diff --git a/engine/database/models.cc b/engine/database/models.cc
index c4365eb68..46d044451 100644
--- a/engine/database/models.cc
+++ b/engine/database/models.cc
@@ -60,10 +60,11 @@ cpp::result<std::vector<ModelEntry>, std::string> Models::LoadModelListNoLock()
     const {
   try {
     std::vector<ModelEntry> entries;
-    SQLite::Statement query(db_,
-                            "SELECT model_id, author_repo_id, branch_name, "
-                            "path_to_model_yaml, model_alias, model_format, "
-                            "model_source, status, engine, metadata FROM models");
+    SQLite::Statement query(
+        db_,
+        "SELECT model_id, author_repo_id, branch_name, "
+        "path_to_model_yaml, model_alias, model_format, "
+        "model_source, status, engine, metadata FROM models");
 
     while (query.executeStep()) {
       ModelEntry entry;
@@ -89,11 +90,12 @@ cpp::result<std::vector<ModelEntry>, std::string> Models::LoadModelListNoLock()
 cpp::result<ModelEntry, std::string> Models::GetModelInfo(
     const std::string& identifier) const {
   try {
-    SQLite::Statement query(db_,
-                            "SELECT model_id, author_repo_id, branch_name, "
-                            "path_to_model_yaml, model_alias, model_format, "
-                            "model_source, status, engine, metadata FROM models "
-                            "WHERE model_id = ?");
+    SQLite::Statement query(
+        db_,
+        "SELECT model_id, author_repo_id, branch_name, "
+        "path_to_model_yaml, model_alias, model_format, "
+        "model_source, status, engine, metadata FROM models "
+        "WHERE model_id = ?");
 
     query.bind(1, identifier);
     if (query.executeStep()) {
@@ -201,8 +203,7 @@ cpp::result<bool, std::string> Models::DeleteModelEntry(
       return true;
     }
 
-    SQLite::Statement del(
-        db_, "DELETE from models WHERE model_id = ?");
+    SQLite::Statement del(db_, "DELETE from models WHERE model_id = ?");
     del.bind(1, identifier);
     return del.exec() == 1;
   } catch (const std::exception& e) {
@@ -240,8 +241,9 @@ cpp::result<std::vector<std::string>, std::string> Models::FindRelatedModel(
     const std::string& identifier) const {
   try {
     std::vector<std::string> related_models;
-    SQLite::Statement query(
-        db_, "SELECT model_id FROM models WHERE model_id LIKE ?");
+    SQLite::Statement query(db_,
+                            "SELECT model_id FROM models WHERE model_id LIKE ? "
+                            "AND status = \"downloaded\"");
     query.bind(1, "%" + identifier + "%");
 
     while (query.executeStep()) {
@@ -255,9 +257,8 @@ cpp::result<std::vector<std::string>, std::string> Models::FindRelatedModel(
 
 bool Models::HasModel(const std::string& identifier) const {
   try {
-    SQLite::Statement query(
-        db_,
-        "SELECT COUNT(*) FROM models WHERE model_id = ?");
+    SQLite::Statement query(db_,
+                            "SELECT COUNT(*) FROM models WHERE model_id = ?");
     query.bind(1, identifier);
     if (query.executeStep()) {
       return query.getColumn(0).getInt() > 0;
diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc
index 90f3c98ef..15fee15be 100644
--- a/engine/services/model_service.cc
+++ b/engine/services/model_service.cc
@@ -64,16 +64,30 @@ void ParseGguf(const DownloadItem& ggufDownloadItem,
 
   auto author_id = author.has_value() ? author.value() : "cortexso";
   cortex::db::Models modellist_utils_obj;
-  cortex::db::ModelEntry model_entry{
-      .model = ggufDownloadItem.id,
-      .author_repo_id = author_id,
-      .branch_name = branch,
-      .path_to_model_yaml = rel.string(),
-      .model_alias = ggufDownloadItem.id,
-      .status = cortex::db::ModelStatus::Downloaded};
-  auto result = modellist_utils_obj.AddModelEntry(model_entry);
-  if (result.has_error()) {
-    CTL_WRN("Error adding model to modellist: " + result.error());
+  if (!modellist_utils_obj.HasModel(ggufDownloadItem.id)) {
+    cortex::db::ModelEntry model_entry{
+        .model = ggufDownloadItem.id,
+        .author_repo_id = author_id,
+        .branch_name = branch,
+        .path_to_model_yaml = rel.string(),
+        .model_alias = ggufDownloadItem.id,
+        .status = cortex::db::ModelStatus::Downloaded};
+    auto result = modellist_utils_obj.AddModelEntry(model_entry);
+
+    if (result.has_error()) {
+      CTL_ERR("Error adding model to modellist: " + result.error());
+    }
+  } else {
+    if (auto m = modellist_utils_obj.GetModelInfo(ggufDownloadItem.id);
+        m.has_value()) {
+      auto upd_m = m.value();
+      upd_m.status = cortex::db::ModelStatus::Downloaded;
+      if (auto r =
+              modellist_utils_obj.UpdateModelEntry(ggufDownloadItem.id, upd_m);
+          r.has_error()) {
+        CTL_ERR(r.error());
+      }
+    }
   }
 }
 
@@ -136,7 +150,7 @@ void ModelService::ForceIndexingModelList() {
 
   CTL_DBG("Database model size: " + std::to_string(list_entry.value().size()));
   for (const auto& model_entry : list_entry.value()) {
-    if(model_entry.status != cortex::db::ModelStatus::Downloaded) {
+    if (model_entry.status != cortex::db::ModelStatus::Downloaded) {
       continue;
     }
     try {
@@ -304,7 +318,8 @@ cpp::result<DownloadTask, std::string> ModelService::HandleDownloadUrlAsync(
   }
 
   auto model_entry = modellist_handler.GetModelInfo(unique_model_id);
-  if (model_entry.has_value()) {
+  if (model_entry.has_value() &&
+      model_entry->status == cortex::db::ModelStatus::Downloaded) {
     CLI_LOG("Model already downloaded: " << unique_model_id);
     return cpp::fail("Please delete the model before downloading again");
   }
@@ -494,7 +509,8 @@ ModelService::DownloadModelFromCortexsoAsync(
   }
 
   auto model_entry = modellist_handler.GetModelInfo(unique_model_id);
-  if (model_entry.has_value()) {
+  if (model_entry.has_value() &&
+      model_entry->status == cortex::db::ModelStatus::Downloaded) {
     return cpp::fail("Please delete the model before downloading again");
   }
 
@@ -535,14 +551,32 @@ ModelService::DownloadModelFromCortexsoAsync(
     CTL_INF("path_to_model_yaml: " << rel.string());
 
     cortex::db::Models modellist_utils_obj;
-    cortex::db::ModelEntry model_entry{.model = unique_model_id,
-                                       .author_repo_id = "cortexso",
-                                       .branch_name = branch,
-                                       .path_to_model_yaml = rel.string(),
-                                       .model_alias = unique_model_id};
-    auto result = modellist_utils_obj.AddModelEntry(model_entry);
-    if (result.has_error()) {
-      CTL_ERR("Error adding model to modellist: " + result.error());
+    if (!modellist_utils_obj.HasModel(unique_model_id)) {
+      cortex::db::ModelEntry model_entry{
+          .model = unique_model_id,
+          .author_repo_id = "cortexso",
+          .branch_name = branch,
+          .path_to_model_yaml = rel.string(),
+          .model_alias = unique_model_id,
+          .status = cortex::db::ModelStatus::Downloaded};
+      auto result = modellist_utils_obj.AddModelEntry(model_entry);
+
+      if (result.has_error()) {
+        CTL_ERR("Error adding model to modellist: " + result.error());
+      }
+    } else {
+      if (auto m = modellist_utils_obj.GetModelInfo(unique_model_id);
+          m.has_value()) {
+        auto upd_m = m.value();
+        upd_m.status = cortex::db::ModelStatus::Downloaded;
+        if (auto r =
+                modellist_utils_obj.UpdateModelEntry(unique_model_id, upd_m);
+            r.has_error()) {
+          CTL_ERR(r.error());
+        }
+      } else {
+        CTL_WRN("Could not get model entry with model id: " << unique_model_id);
+      }
     }
   };
 
@@ -588,14 +622,28 @@ cpp::result<std::string, std::string> ModelService::DownloadModelFromCortexso(
     CTL_INF("path_to_model_yaml: " << rel.string());
 
     cortex::db::Models modellist_utils_obj;
-    cortex::db::ModelEntry model_entry{.model = model_id,
-                                       .author_repo_id = "cortexso",
-                                       .branch_name = branch,
-                                       .path_to_model_yaml = rel.string(),
-                                       .model_alias = model_id};
-    auto result = modellist_utils_obj.AddModelEntry(model_entry);
-    if (result.has_error()) {
-      CTL_ERR("Error adding model to modellist: " + result.error());
+    if (!modellist_utils_obj.HasModel(model_id)) {
+      cortex::db::ModelEntry model_entry{
+          .model = model_id,
+          .author_repo_id = "cortexso",
+          .branch_name = branch,
+          .path_to_model_yaml = rel.string(),
+          .model_alias = model_id,
+          .status = cortex::db::ModelStatus::Downloaded};
+      auto result = modellist_utils_obj.AddModelEntry(model_entry);
+
+      if (result.has_error()) {
+        CTL_ERR("Error adding model to modellist: " + result.error());
+      }
+    } else {
+      if (auto m = modellist_utils_obj.GetModelInfo(model_id); m.has_value()) {
+        auto upd_m = m.value();
+        upd_m.status = cortex::db::ModelStatus::Downloaded;
+        if (auto r = modellist_utils_obj.UpdateModelEntry(model_id, upd_m);
+            r.has_error()) {
+          CTL_ERR(r.error());
+        }
+      }
     }
   };
 
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index c40e2b724..571632863 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -179,7 +179,10 @@ cpp::result<bool, std::string> ModelSourceService::AddHfOrg(
     // Clean up
     for (auto const& mid : model_list_before) {
       if (updated_model_list.find(mid) == updated_model_list.end()) {
-        (void)model_db.DeleteModelEntry(mid);
+        if (auto del_res = model_db.DeleteModelEntry(mid);
+            del_res.has_error()) {
+          CTL_INF(del_res.error());
+        }
       }
     }
   } else {
@@ -205,7 +208,9 @@ cpp::result<bool, std::string> ModelSourceService::AddHfRepo(
   }
   for (auto const& mid : model_list_before) {
     if (updated_model_list.find(mid) == updated_model_list.end()) {
-      (void)model_db.DeleteModelEntry(mid);
+      if (auto del_res = model_db.DeleteModelEntry(mid); del_res.has_error()) {
+        CTL_INF(del_res.error());
+      }
     }
   }
   return true;
@@ -244,9 +249,18 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source,
           .engine = "llama-cpp",
           .metadata = repo_info->metadata};
       if (!model_db.HasModel(model_id)) {
-        (void)model_db.AddModelEntry(e);
+        if (auto add_res = model_db.AddModelEntry(e); add_res.has_error()) {
+          CTL_INF(add_res.error());
+        }
       } else {
-        (void)model_db.UpdateModelEntry(model_id, e);
+        if (auto m = model_db.GetModelInfo(model_id);
+            m.has_value() &&
+            m->status == cortex::db::ModelStatus::Undownloaded) {
+          if (auto upd_res = model_db.UpdateModelEntry(model_id, e);
+              upd_res.has_error()) {
+            CTL_INF(upd_res.error());
+          }
+        }
       }
       res.insert(model_id);
     }
@@ -299,7 +313,10 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoOrg(
     // Clean up
     for (auto const& mid : model_list_before) {
       if (updated_model_list.find(mid) == updated_model_list.end()) {
-        (void)model_db.DeleteModelEntry(mid);
+        if (auto del_res = model_db.DeleteModelEntry(mid);
+            del_res.has_error()) {
+          CTL_INF(del_res.error());
+        }
       }
     }
   } else {
@@ -342,7 +359,9 @@ cpp::result<bool, std::string> ModelSourceService::AddCortexsoRepo(
   // Clean up
   for (auto const& mid : model_list_before) {
     if (updated_model_list.find(mid) == updated_model_list.end()) {
-      (void)model_db.DeleteModelEntry(mid);
+      if (auto del_res = model_db.DeleteModelEntry(mid); del_res.has_error()) {
+        CTL_INF(del_res.error());
+      }
     }
   }
   return true;
@@ -397,8 +416,13 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source,
         CTL_DBG("Cannot add model to db: " << model_id);
       }
     } else {
-      (void)model_db.UpdateModelEntry(model_id, e);
-      CTL_DBG("Updated model: " << model_id);
+      if (auto m = model_db.GetModelInfo(model_id);
+          m.has_value() && m->status == cortex::db::ModelStatus::Undownloaded) {
+        if (auto upd_res = model_db.UpdateModelEntry(model_id, e);
+            upd_res.has_error()) {
+          CTL_INF(upd_res.error());
+        }
+      }
     }
     res.insert(model_id);
   }

From 1181426caa4af10cb9482ab2f293dbad7550fd71 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Mon, 9 Dec 2024 14:35:57 +0700
Subject: [PATCH 18/21] chore: unit tests: update

---
 engine/test/components/test_models_db.cc | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/engine/test/components/test_models_db.cc b/engine/test/components/test_models_db.cc
index decf8b769..06294aa8c 100644
--- a/engine/test/components/test_models_db.cc
+++ b/engine/test/components/test_models_db.cc
@@ -24,7 +24,8 @@ class ModelsTestSuite : public ::testing::Test {
           "model_format TEXT,"
           "model_source TEXT,"
           "status TEXT,"
-          "engine TEXT"
+          "engine TEXT,"
+          "metadata TEXT"
           ")");
     } catch (const std::exception& e) {}
   }
@@ -70,10 +71,6 @@ TEST_F(ModelsTestSuite, TestGetModelInfo) {
   EXPECT_TRUE(model_by_id.has_value());
   EXPECT_EQ(model_by_id.value().model, kTestModel.model);
 
-  auto model_by_alias = model_list_.GetModelInfo("test_alias");
-  EXPECT_TRUE(model_by_alias);
-  EXPECT_EQ(model_by_alias.value().model, kTestModel.model);
-
   EXPECT_TRUE(model_list_.GetModelInfo("non_existent_model").has_error());
 
   // Clean up
@@ -120,7 +117,6 @@ TEST_F(ModelsTestSuite, TestHasModel) {
   EXPECT_TRUE(model_list_.AddModelEntry(kTestModel).value());
 
   EXPECT_TRUE(model_list_.HasModel(kTestModel.model));
-  EXPECT_TRUE(model_list_.HasModel("test_alias"));
   EXPECT_FALSE(model_list_.HasModel("non_existent_model"));
   // Clean up
   EXPECT_TRUE(model_list_.DeleteModelEntry(kTestModel.model).value());

From 67636c8265930ebda2511a542bd4b6ac07bd3911 Mon Sep 17 00:00:00 2001
From: vansangpfiev <sang@jan.ai>
Date: Tue, 10 Dec 2024 09:44:14 +0700
Subject: [PATCH 19/21] chore: add e2e tests for models sources

---
 engine/e2e-test/test_api_model.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/engine/e2e-test/test_api_model.py b/engine/e2e-test/test_api_model.py
index c2723d2ca..8f2e4b07a 100644
--- a/engine/e2e-test/test_api_model.py
+++ b/engine/e2e-test/test_api_model.py
@@ -129,4 +129,17 @@ async def test_models_start_stop_should_be_successful(self):
         # delete API
         print("Delete model")
         response = requests.delete("http://localhost:3928/v1/models/tinyllama:gguf")
-        assert response.status_code == 200
\ No newline at end of file
+        assert response.status_code == 200
+        
+    def test_models_sources_api(self):
+        json_body = {"source": "https://huggingface.co/cortexso/tinyllama"}
+        response = requests.post(
+            "http://localhost:3928/v1/models/sources", json=json_body
+        )
+        assert response.status_code == 200, f"status_code: {response.status_code}"
+        
+        json_body = {"source": "https://huggingface.co/cortexso/tinyllama"}
+        response = requests.delete(
+            "http://localhost:3928/v1/models/sources", json=json_body
+        )
+        assert response.status_code == 200, f"status_code: {response.status_code}"
\ No newline at end of file

From 53664a98f1c9fa6a2fcc950260bca9294f0fdc2b Mon Sep 17 00:00:00 2001
From: vansangpfiev <vansangpfiev@gmail.com>
Date: Wed, 11 Dec 2024 07:02:56 +0700
Subject: [PATCH 20/21] chore: add API docs

---
 docs/static/openapi/cortex.json | 99 +++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)

diff --git a/docs/static/openapi/cortex.json b/docs/static/openapi/cortex.json
index 9cdd5c7b4..2ff239ce2 100644
--- a/docs/static/openapi/cortex.json
+++ b/docs/static/openapi/cortex.json
@@ -807,6 +807,105 @@
         "tags": ["Pulling Models"]
       }
     },
+    "/v1/models/sources": {
+      "post": {
+        "summary": "Add a model source",
+        "description": "User can add a Huggingface Organization or Repository",        
+        "requestBody": {
+          "required": false,
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "source": {
+                    "type": "string",
+                    "description": "The url of model source to add",
+                    "example": "https://huggingface.co/cortexso/tinyllama"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Successful installation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "message": {
+                      "type": "string",
+                      "example": "Added model source"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "tags": ["Pulling Models"]
+      },
+      "delete": {
+        "summary": "Remove a model source",
+        "description": "User can remove a Huggingface Organization or Repository",        
+        "requestBody": {
+          "required": false,
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "source": {
+                    "type": "string",
+                     "description": "The url of model source to remove",
+                    "example": "https://huggingface.co/cortexso/tinyllama"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Successful uninstallation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "message": {
+                      "type": "string",
+                      "description": "Removed model source successfully!",
+                      "example": "Removed model source successfully!"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Bad request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "description": "Error message describing the issue with the request"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "tags": ["Pulling Models"]
+      }
+    },
     "/v1/threads": {
       "post": {
         "operationId": "ThreadsController_create",

From 92f05976666b892026041c767cea09b616355352 Mon Sep 17 00:00:00 2001
From: vansangpfiev <vansangpfiev@gmail.com>
Date: Wed, 11 Dec 2024 08:12:00 +0700
Subject: [PATCH 21/21] chore: rename

---
 engine/cli/commands/model_list_cmd.cc   |  4 ++--
 engine/controllers/models.cc            |  6 +++---
 engine/database/models.cc               | 16 ++++++++--------
 engine/database/models.h                |  2 +-
 engine/services/model_source_service.cc |  8 ++++----
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/engine/cli/commands/model_list_cmd.cc b/engine/cli/commands/model_list_cmd.cc
index 03275f7fd..96ff2885d 100644
--- a/engine/cli/commands/model_list_cmd.cc
+++ b/engine/cli/commands/model_list_cmd.cc
@@ -74,7 +74,7 @@ void ModelListCmd::Exec(const std::string& host, int port,
       }
 
       if (available) {
-        if (v["status"].asString() != "undownloaded") {
+        if (v["status"].asString() != "downloadable") {
           continue;
         }
 
@@ -90,7 +90,7 @@ void ModelListCmd::Exec(const std::string& host, int port,
         }
         table.add_row({row.begin(), row.end()});
       } else {
-        if (v["status"].asString() == "undownloaded") {
+        if (v["status"].asString() == "downloadable") {
           continue;
         }
 
diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc
index 8fe647eab..affa45d52 100644
--- a/engine/controllers/models.cc
+++ b/engine/controllers/models.cc
@@ -172,7 +172,7 @@ void Models::ListModel(
   if (list_entry) {
     for (const auto& model_entry : list_entry.value()) {
       try {
-        if (model_entry.status == cortex::db::ModelStatus::Undownloaded) {
+        if (model_entry.status == cortex::db::ModelStatus::Downloadable) {
           Json::Value obj;
           obj["id"] = model_entry.model;
           obj["model"] = model_entry.model;
@@ -182,8 +182,8 @@ void Models::ListModel(
                 return "remote";
               case cortex::db::ModelStatus::Downloaded:
                 return "downloaded";
-              case cortex::db::ModelStatus::Undownloaded:
-                return "undownloaded";
+              case cortex::db::ModelStatus::Downloadable:
+                return "downloadable";
             }
             return "unknown";
           };
diff --git a/engine/database/models.cc b/engine/database/models.cc
index 46d044451..67ff1a8c9 100644
--- a/engine/database/models.cc
+++ b/engine/database/models.cc
@@ -18,8 +18,8 @@ std::string Models::StatusToString(ModelStatus status) const {
       return "remote";
     case ModelStatus::Downloaded:
       return "downloaded";
-    case ModelStatus::Undownloaded:
-      return "undownloaded";
+    case ModelStatus::Downloadable:
+      return "downloadable";
   }
   return "unknown";
 }
@@ -31,8 +31,8 @@ ModelStatus Models::StringToStatus(const std::string& status_str) const {
     return ModelStatus::Remote;
   } else if (status_str == "downloaded" || status_str.empty()) {
     return ModelStatus::Downloaded;
-  } else if (status_str == "undownloaded") {
-    return ModelStatus::Undownloaded;
+  } else if (status_str == "downloadable") {
+    return ModelStatus::Downloadable;
   }
   throw std::invalid_argument("Invalid status string");
 }
@@ -216,7 +216,7 @@ cpp::result<bool, std::string> Models::DeleteModelEntryWithOrg(
   try {
     SQLite::Statement del(db_,
                           "DELETE from models WHERE model_source LIKE ? AND "
-                          "status = \"undownloaded\"");
+                          "status = \"downloadable\"");
     del.bind(1, src + "%");
     return del.exec() == 1;
   } catch (const std::exception& e) {
@@ -229,7 +229,7 @@ cpp::result<bool, std::string> Models::DeleteModelEntryWithRepo(
   try {
     SQLite::Statement del(db_,
                           "DELETE from models WHERE model_source = ? AND "
-                          "status = \"undownloaded\"");
+                          "status = \"downloadable\"");
     del.bind(1, src);
     return del.exec() == 1;
   } catch (const std::exception& e) {
@@ -276,7 +276,7 @@ cpp::result<std::vector<std::string>, std::string> Models::GetModelSources()
     std::vector<std::string> sources;
     SQLite::Statement query(db_,
                             "SELECT DISTINCT model_source FROM models WHERE "
-                            "status = \"undownloaded\"");
+                            "status = \"downloadable\"");
 
     while (query.executeStep()) {
       sources.push_back(query.getColumn(0).getString());
@@ -293,7 +293,7 @@ cpp::result<std::vector<std::string>, std::string> Models::GetModels(
     std::vector<std::string> ids;
     SQLite::Statement query(db_,
                             "SELECT model_id FROM models WHERE model_source = "
-                            "? AND status = \"undownloaded\"");
+                            "? AND status = \"downloadable\"");
     query.bind(1, model_src);
     while (query.executeStep()) {
       ids.push_back(query.getColumn(0).getString());
diff --git a/engine/database/models.h b/engine/database/models.h
index 034acd825..74d3937c8 100644
--- a/engine/database/models.h
+++ b/engine/database/models.h
@@ -8,7 +8,7 @@
 
 namespace cortex::db {
 
-enum class ModelStatus { Remote, Downloaded, Undownloaded };
+enum class ModelStatus { Remote, Downloaded, Downloadable };
 
 struct ModelEntry {
   std::string model;
diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc
index 571632863..a7d9d5e6e 100644
--- a/engine/services/model_source_service.cc
+++ b/engine/services/model_source_service.cc
@@ -245,7 +245,7 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source,
           .model_alias = "",
           .model_format = "hf-gguf",
           .model_source = model_source,
-          .status = cortex::db::ModelStatus::Undownloaded,
+          .status = cortex::db::ModelStatus::Downloadable,
           .engine = "llama-cpp",
           .metadata = repo_info->metadata};
       if (!model_db.HasModel(model_id)) {
@@ -255,7 +255,7 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source,
       } else {
         if (auto m = model_db.GetModelInfo(model_id);
             m.has_value() &&
-            m->status == cortex::db::ModelStatus::Undownloaded) {
+            m->status == cortex::db::ModelStatus::Downloadable) {
           if (auto upd_res = model_db.UpdateModelEntry(model_id, e);
               upd_res.has_error()) {
             CTL_INF(upd_res.error());
@@ -406,7 +406,7 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source,
                                 .model_alias = "",
                                 .model_format = "cortexso",
                                 .model_source = model_source,
-                                .status = cortex::db::ModelStatus::Undownloaded,
+                                .status = cortex::db::ModelStatus::Downloadable,
                                 .engine = "llama-cpp",
                                 .metadata = metadata};
     if (!model_db.HasModel(model_id)) {
@@ -417,7 +417,7 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source,
       }
     } else {
       if (auto m = model_db.GetModelInfo(model_id);
-          m.has_value() && m->status == cortex::db::ModelStatus::Undownloaded) {
+          m.has_value() && m->status == cortex::db::ModelStatus::Downloadable) {
         if (auto upd_res = model_db.UpdateModelEntry(model_id, e);
             upd_res.has_error()) {
           CTL_INF(upd_res.error());