diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index 98cf8be51..fb9cc7e22 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -58,6 +58,20 @@ void Models::PullModel(const HttpRequestPtr& req, model_handle, desired_model_id, desired_model_name); } else if (model_handle.find(":") != std::string::npos) { auto model_and_branch = string_utils::SplitBy(model_handle, ":"); + if (model_and_branch.size() == 3) { + auto mh = url_parser::Url{ + .protocol = "https", + .host = kHuggingFaceHost, + .pathParams = { + model_and_branch[0], + model_and_branch[1], + "resolve", + "main", + model_and_branch[2], + }}.ToFullPath(); + return model_service_->HandleDownloadUrlAsync(mh, desired_model_id, + desired_model_name); + } return model_service_->DownloadModelFromCortexsoAsync( model_and_branch[0], model_and_branch[1], desired_model_id); } @@ -813,15 +827,34 @@ void Models::GetModelSources( resp->setStatusCode(k400BadRequest); callback(resp); } else { - auto const& info = res.value(); + auto& info = res.value(); Json::Value ret; Json::Value data(Json::arrayValue); - for (auto const& i : info) { - data.append(i); + for (auto& i : info) { + data.append(i.second.ToJson()); } ret["data"] = data; auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); resp->setStatusCode(k200OK); callback(resp); } +} + +void Models::GetModelSource( + const HttpRequestPtr& req, + std::function&& callback, + const std::string& src) { + auto res = model_src_svc_->GetModelSource(src); + if (res.has_error()) { + Json::Value ret; + ret["message"] = res.error(); + auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); + resp->setStatusCode(k400BadRequest); + callback(resp); + } else { + auto& info = res.value(); + auto resp = cortex_utils::CreateCortexHttpJsonResponse(info.ToJson()); + resp->setStatusCode(k200OK); + callback(resp); + } } \ No newline at end of file diff --git a/engine/controllers/models.h b/engine/controllers/models.h index 60053acdb..8f6af4280 100644 --- a/engine/controllers/models.h +++ b/engine/controllers/models.h @@ -43,6 +43,7 @@ class Models : public drogon::HttpController { ADD_METHOD_TO(Models::AddModelSource, "/v1/models/sources", Post); ADD_METHOD_TO(Models::DeleteModelSource, "/v1/models/sources", Delete); ADD_METHOD_TO(Models::GetModelSources, "/v1/models/sources", Get); + ADD_METHOD_TO(Models::GetModelSource, "/v1/models/sources/{src}", Get); METHOD_LIST_END explicit Models(std::shared_ptr db_service, @@ -106,6 +107,10 @@ class Models : public drogon::HttpController { void GetModelSources(const HttpRequestPtr& req, std::function&& callback); + void GetModelSource(const HttpRequestPtr& req, + std::function&& callback, + const std::string& src); + private: std::shared_ptr db_service_; std::shared_ptr model_service_; diff --git a/engine/database/models.cc b/engine/database/models.cc index 67ff1a8c9..6bf891040 100644 --- a/engine/database/models.cc +++ b/engine/database/models.cc @@ -270,35 +270,63 @@ bool Models::HasModel(const std::string& identifier) const { } } -cpp::result, std::string> Models::GetModelSources() - const { +cpp::result, std::string> Models::GetModels( + const std::string& model_src) const { try { - std::vector sources; + std::vector res; SQLite::Statement query(db_, - "SELECT DISTINCT model_source FROM models WHERE " - "status = \"downloadable\""); - + "SELECT model_id, author_repo_id, branch_name, " + "path_to_model_yaml, model_alias, model_format, " + "model_source, status, engine, metadata FROM " + "models WHERE model_source = " + "? AND status = \"downloadable\""); + query.bind(1, model_src); while (query.executeStep()) { - sources.push_back(query.getColumn(0).getString()); + ModelEntry entry; + entry.model = query.getColumn(0).getString(); + entry.author_repo_id = query.getColumn(1).getString(); + entry.branch_name = query.getColumn(2).getString(); + entry.path_to_model_yaml = query.getColumn(3).getString(); + entry.model_alias = query.getColumn(4).getString(); + entry.model_format = query.getColumn(5).getString(); + entry.model_source = query.getColumn(6).getString(); + entry.status = StringToStatus(query.getColumn(7).getString()); + entry.engine = query.getColumn(8).getString(); + entry.metadata = query.getColumn(9).getString(); + res.push_back(entry); } - return sources; + return res; } catch (const std::exception& e) { return cpp::fail(e.what()); } } -cpp::result, std::string> Models::GetModels( - const std::string& model_src) const { +cpp::result, std::string> Models::GetModelSources() + const { try { - std::vector ids; - SQLite::Statement query(db_, - "SELECT model_id FROM models WHERE model_source = " - "? AND status = \"downloadable\""); - query.bind(1, model_src); + std::vector res; + SQLite::Statement query( + db_, + "SELECT model_id, author_repo_id, branch_name, " + "path_to_model_yaml, model_alias, model_format, " + "model_source, status, engine, metadata FROM models " + "WHERE model_source != \"\" AND (status = \"downloaded\" OR status = " + "\"downloadable\")"); while (query.executeStep()) { - ids.push_back(query.getColumn(0).getString()); + ModelEntry entry; + entry.model = query.getColumn(0).getString(); + entry.author_repo_id = query.getColumn(1).getString(); + entry.branch_name = query.getColumn(2).getString(); + entry.path_to_model_yaml = query.getColumn(3).getString(); + entry.model_alias = query.getColumn(4).getString(); + entry.model_format = query.getColumn(5).getString(); + entry.model_source = query.getColumn(6).getString(); + entry.status = StringToStatus(query.getColumn(7).getString()); + entry.engine = query.getColumn(8).getString(); + entry.metadata = query.getColumn(9).getString(); + res.push_back(entry); } - return ids; + return res; } catch (const std::exception& e) { return cpp::fail(e.what()); } diff --git a/engine/database/models.h b/engine/database/models.h index b0c4bc258..b0059dbea 100644 --- a/engine/database/models.h +++ b/engine/database/models.h @@ -10,7 +10,6 @@ namespace cortex::db { enum class ModelStatus { Remote, Downloaded, Downloadable }; - struct ModelEntry { std::string model; std::string author_repo_id; @@ -57,9 +56,9 @@ class Models { cpp::result, std::string> FindRelatedModel( const std::string& identifier) const; bool HasModel(const std::string& identifier) const; - cpp::result, std::string> GetModelSources() const; - cpp::result, std::string> GetModels( + cpp::result, std::string> GetModels( const std::string& model_src) const; + cpp::result, std::string> GetModelSources() const; }; } // namespace cortex::db diff --git a/engine/services/database_service.cc b/engine/services/database_service.cc index d4cd977a9..695e36f72 100644 --- a/engine/services/database_service.cc +++ b/engine/services/database_service.cc @@ -118,13 +118,13 @@ bool DatabaseService::HasModel(const std::string& identifier) const { return cortex::db::Models().HasModel(identifier); } -cpp::result, std::string> -DatabaseService::GetModelSources() const { - return cortex::db::Models().GetModelSources(); -} - -cpp::result, std::string> DatabaseService::GetModels( +cpp::result, std::string> DatabaseService::GetModels( const std::string& model_src) const { return cortex::db::Models().GetModels(model_src); } + +cpp::result, std::string> +DatabaseService::GetModelSources() const { + return cortex::db::Models().GetModelSources(); +} // end models \ No newline at end of file diff --git a/engine/services/database_service.h b/engine/services/database_service.h index 4fb4f7be0..e45fc57b2 100644 --- a/engine/services/database_service.h +++ b/engine/services/database_service.h @@ -60,9 +60,10 @@ class DatabaseService { cpp::result, std::string> FindRelatedModel( const std::string& identifier) const; bool HasModel(const std::string& identifier) const; - cpp::result, std::string> GetModelSources() const; - cpp::result, std::string> GetModels( + cpp::result, std::string> GetModels( const std::string& model_src) const; + cpp::result, std::string> GetModelSources() + const; private: }; \ No newline at end of file diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index 4a21dff40..6dc1642fb 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -67,7 +67,8 @@ void ParseGguf(DatabaseService& db_service, CTL_INF("Adding model to modellist with branch: " << branch); auto rel = file_manager_utils::ToRelativeCortexDataPath(yaml_name); - CTL_INF("path_to_model_yaml: " << rel.string()); + CTL_INF("path_to_model_yaml: " << rel.string() + << ", model: " << ggufDownloadItem.id); auto author_id = author.has_value() ? author.value() : "cortexso"; if (!db_service.HasModel(ggufDownloadItem.id)) { @@ -86,6 +87,7 @@ void ParseGguf(DatabaseService& db_service, } else { if (auto m = db_service.GetModelInfo(ggufDownloadItem.id); m.has_value()) { auto upd_m = m.value(); + upd_m.path_to_model_yaml = rel.string(); upd_m.status = cortex::db::ModelStatus::Downloaded; if (auto r = db_service.UpdateModelEntry(ggufDownloadItem.id, upd_m); r.has_error()) { @@ -161,6 +163,9 @@ void ModelService::ForceIndexingModelList() { continue; } try { + CTL_DBG(fmu::ToAbsoluteCortexDataPath( + fs::path(model_entry.path_to_model_yaml)) + .string()); yaml_handler.ModelConfigFromFile( fmu::ToAbsoluteCortexDataPath( fs::path(model_entry.path_to_model_yaml)) @@ -171,48 +176,12 @@ void ModelService::ForceIndexingModelList() { } catch (const std::exception& e) { // remove in db auto remove_result = db_service_->DeleteModelEntry(model_entry.model); + CTL_DBG(e.what()); // silently ignore result } } } -cpp::result ModelService::DownloadModel( - const std::string& input) { - if (input.empty()) { - return cpp::fail( - "Input must be Cortex Model Hub handle or HuggingFace url!"); - } - - if (string_utils::StartsWith(input, "https://")) { - return HandleUrl(input); - } - - if (input.find(":") != std::string::npos) { - auto parsed = string_utils::SplitBy(input, ":"); - if (parsed.size() != 2) { - return cpp::fail("Invalid model handle: " + input); - } - return DownloadModelFromCortexso(parsed[0], parsed[1]); - } - - if (input.find("/") != std::string::npos) { - auto parsed = string_utils::SplitBy(input, "/"); - if (parsed.size() != 2) { - return cpp::fail("Invalid model handle: " + input); - } - - auto author = parsed[0]; - auto model_name = parsed[1]; - if (author == "cortexso") { - return HandleCortexsoModel(model_name); - } - - return DownloadHuggingFaceGgufModel(author, model_name, std::nullopt); - } - - return HandleCortexsoModel(input); -} - cpp::result ModelService::HandleCortexsoModel( const std::string& modelName) { auto branches = @@ -612,7 +581,8 @@ ModelService::DownloadModelFromCortexsoAsync( .branch_name = branch, .path_to_model_yaml = rel.string(), .model_alias = unique_model_id, - .status = cortex::db::ModelStatus::Downloaded}; + .status = cortex::db::ModelStatus::Downloaded, + .engine = mc.engine}; auto result = db_service_->AddModelEntry(model_entry); if (result.has_error()) { @@ -621,6 +591,7 @@ ModelService::DownloadModelFromCortexsoAsync( } else { if (auto m = db_service_->GetModelInfo(unique_model_id); m.has_value()) { auto upd_m = m.value(); + upd_m.path_to_model_yaml = rel.string(); upd_m.status = cortex::db::ModelStatus::Downloaded; if (auto r = db_service_->UpdateModelEntry(unique_model_id, upd_m); r.has_error()) { @@ -1157,7 +1128,7 @@ cpp::result ModelService::GetModelPullInfo( if (input.find(":") != std::string::npos) { auto parsed = string_utils::SplitBy(input, ":"); - if (parsed.size() != 2) { + if (parsed.size() != 2 && parsed.size() != 3) { return cpp::fail("Invalid model handle: " + input); } return ModelPullInfo{.id = input, diff --git a/engine/services/model_service.h b/engine/services/model_service.h index a668b27ba..17f2c0ddb 100644 --- a/engine/services/model_service.h +++ b/engine/services/model_service.h @@ -42,11 +42,6 @@ class ModelService { inference_svc_(inference_service), engine_svc_(engine_svc) {}; - /** - * Return model id if download successfully - */ - cpp::result DownloadModel(const std::string& input); - cpp::result AbortDownloadModel( const std::string& task_id); diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc index 7fc0ef5b2..aaf8b469b 100644 --- a/engine/services/model_source_service.cc +++ b/engine/services/model_source_service.cc @@ -4,6 +4,7 @@ #include "database/models.h" #include "json/json.h" #include "utils/curl_utils.h" +#include "utils/file_manager_utils.h" #include "utils/huggingface_utils.h" #include "utils/logging_utils.h" #include "utils/string_utils.h" @@ -113,7 +114,14 @@ cpp::result ModelSourceService::RemoveModelSource( return cpp::fail(srcs.error()); } else { auto& v = srcs.value(); - if (std::find(v.begin(), v.end(), model_source) == v.end()) { + auto exists = [&v, &model_source]() { + for (auto const& m : v) { + if (m.model_source == model_source) + return true; + } + return false; + }(); + if (!exists) { return cpp::fail("Model source does not exist: " + model_source); } } @@ -144,9 +152,50 @@ cpp::result ModelSourceService::RemoveModelSource( return true; } -cpp::result, std::string> +cpp::result, std::string> ModelSourceService::GetModelSources() { - return db_service_->GetModelSources(); + auto res = db_service_->GetModelSources(); + if (res.has_error()) { + return cpp::fail(res.error()); + } + auto& models = res.value(); + std::unordered_map ms; + for (auto const& m : models) { + auto meta_json = json_helper::ParseJsonString(m.metadata); + ms[m.model_source].models.push_back( + {m.model, meta_json["size"].asUInt64()}); + meta_json.removeMember("size"); + if (ms[m.model_source].metadata.isNull()) { + ms[m.model_source].metadata = meta_json; + } + ms[m.model_source].id = m.model_source; + ms[m.model_source].author = m.author_repo_id; + LOG_DEBUG << m.model; + } + return ms; +} + +cpp::result ModelSourceService::GetModelSource( + const std::string& src) { + auto res = db_service_->GetModels(src); + if (res.has_error()) { + return cpp::fail(res.error()); + } + + auto& models = res.value(); + ModelSource ms; + for (auto const& m : models) { + auto meta_json = json_helper::ParseJsonString(m.metadata); + ms.models.push_back({m.model, meta_json["size"].asUInt64()}); + meta_json.removeMember("size"); + if (ms.metadata.isNull()) { + ms.metadata = meta_json; + } + ms.id = m.model_source; + ms.author = m.author_repo_id; + LOG_INFO << m.model; + } + return ms; } cpp::result ModelSourceService::AddHfOrg( @@ -155,32 +204,17 @@ cpp::result ModelSourceService::AddHfOrg( author); if (res.has_value()) { auto models = ParseJsonString(res.value()); - // Get models from db - - auto model_list_before = db_service_->GetModels(model_source) - .value_or(std::vector{}); - std::unordered_set updated_model_list; // Add new models for (auto const& m : models) { CTL_DBG(m.id); + auto author_model = string_utils::SplitBy(m.id, "/"); if (author_model.size() == 2) { auto const& author = author_model[0]; auto const& model_name = author_model[1]; - auto add_res = AddRepoSiblings(model_source, author, model_name) - .value_or(std::unordered_set{}); - for (auto const& a : add_res) { - updated_model_list.insert(a); - } - } - } - - // Clean up - for (auto const& mid : model_list_before) { - if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = db_service_->DeleteModelEntry(mid); - del_res.has_error()) { - CTL_INF(del_res.error()); + auto r = AddHfRepo(model_source + "/" + model_name, author, model_name); + if (r.has_error()) { + CTL_WRN(r.error()); } } } @@ -195,8 +229,8 @@ cpp::result ModelSourceService::AddHfRepo( const std::string& model_name) { // Get models from db - auto model_list_before = - db_service_->GetModels(model_source).value_or(std::vector{}); + auto model_list_before = db_service_->GetModels(model_source) + .value_or(std::vector{}); std::unordered_set updated_model_list; auto add_res = AddRepoSiblings(model_source, author, model_name); if (add_res.has_error()) { @@ -205,8 +239,8 @@ cpp::result ModelSourceService::AddHfRepo( updated_model_list = add_res.value(); } for (auto const& mid : model_list_before) { - if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = db_service_->DeleteModelEntry(mid); + if (updated_model_list.find(mid.model) == updated_model_list.end()) { + if (auto del_res = db_service_->DeleteModelEntry(mid.model); del_res.has_error()) { CTL_INF(del_res.error()); } @@ -231,8 +265,38 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source, "supported."); } + auto siblings_fs = hu::GetSiblingsFileSize(author, model_name); + + if (siblings_fs.has_error()) { + return cpp::fail("Could not get siblings file size: " + author + "/" + + model_name); + } + + auto readme = hu::GetReadMe(author, model_name); + std::string desc; + if (!readme.has_error()) { + desc = readme.value(); + } + + auto meta_json = json_helper::ParseJsonString(repo_info->metadata); + auto& siblings_fs_v = siblings_fs.value(); + for (auto& m : meta_json["siblings"]) { + auto r_file = m["rfilename"].asString(); + if (siblings_fs_v.file_sizes.find(r_file) != + siblings_fs_v.file_sizes.end()) { + m["size"] = siblings_fs_v.file_sizes.at(r_file).size_in_bytes; + } + } + meta_json["description"] = desc; + LOG_DEBUG << meta_json.toStyledString(); + for (const auto& sibling : repo_info->siblings) { if (string_utils::EndsWith(sibling.rfilename, ".gguf")) { + if (siblings_fs_v.file_sizes.find(sibling.rfilename) != + siblings_fs_v.file_sizes.end()) { + meta_json["size"] = + siblings_fs_v.file_sizes.at(sibling.rfilename).size_in_bytes; + } std::string model_id = author + ":" + model_name + ":" + sibling.rfilename; cortex::db::ModelEntry e = { @@ -245,7 +309,7 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source, .model_source = model_source, .status = cortex::db::ModelStatus::Downloadable, .engine = "llama-cpp", - .metadata = repo_info->metadata}; + .metadata = json_helper::DumpJsonString(meta_json)}; if (!db_service_->HasModel(model_id)) { if (auto add_res = db_service_->AddModelEntry(e); add_res.has_error()) { CTL_INF(add_res.error()); @@ -273,46 +337,16 @@ cpp::result ModelSourceService::AddCortexsoOrg( "https://huggingface.co/api/models?author=cortexso"); if (res.has_value()) { auto models = ParseJsonString(res.value()); - // Get models from db - - auto model_list_before = db_service_->GetModels(model_source) - .value_or(std::vector{}); - std::unordered_set updated_model_list; for (auto const& m : models) { CTL_INF(m.id); auto author_model = string_utils::SplitBy(m.id, "/"); if (author_model.size() == 2) { auto const& author = author_model[0]; auto const& model_name = author_model[1]; - auto branches = huggingface_utils::GetModelRepositoryBranches( - "cortexso", model_name); - if (branches.has_error()) { - CTL_INF(branches.error()); - continue; - } - - auto repo_info = hu::GetHuggingFaceModelRepoInfo(author, model_name); - if (repo_info.has_error()) { - CTL_INF(repo_info.error()); - continue; - } - for (auto const& [branch, _] : branches.value()) { - CTL_INF(branch); - auto add_res = AddCortexsoRepoBranch(model_source, author, model_name, - branch, repo_info->metadata) - .value_or(std::unordered_set{}); - for (auto const& a : add_res) { - updated_model_list.insert(a); - } - } - } - } - // Clean up - for (auto const& mid : model_list_before) { - if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = db_service_->DeleteModelEntry(mid); - del_res.has_error()) { - CTL_INF(del_res.error()); + auto r = AddCortexsoRepo(model_source + "/" + model_name, author, + model_name); + if (r.has_error()) { + CTL_WRN(r.error()); } } } @@ -336,16 +370,22 @@ cpp::result ModelSourceService::AddCortexsoRepo( if (repo_info.has_error()) { return cpp::fail(repo_info.error()); } + + auto readme = hu::GetReadMe(author, model_name); + std::string desc; + if (!readme.has_error()) { + desc = readme.value(); + } // Get models from db - auto model_list_before = - db_service_->GetModels(model_source).value_or(std::vector{}); + auto model_list_before = db_service_->GetModels(model_source) + .value_or(std::vector{}); std::unordered_set updated_model_list; for (auto const& [branch, _] : branches.value()) { CTL_INF(branch); auto add_res = AddCortexsoRepoBranch(model_source, author, model_name, - branch, repo_info->metadata) + branch, repo_info->metadata, desc) .value_or(std::unordered_set{}); for (auto const& a : add_res) { updated_model_list.insert(a); @@ -354,8 +394,8 @@ cpp::result ModelSourceService::AddCortexsoRepo( // Clean up for (auto const& mid : model_list_before) { - if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = db_service_->DeleteModelEntry(mid); + if (updated_model_list.find(mid.model) == updated_model_list.end()) { + if (auto del_res = db_service_->DeleteModelEntry(mid.model); del_res.has_error()) { CTL_INF(del_res.error()); } @@ -369,7 +409,8 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source, const std::string& author, const std::string& model_name, const std::string& branch, - const std::string& metadata) { + const std::string& metadata, + const std::string& desc) { std::unordered_set res; url_parser::Url url = { @@ -384,27 +425,33 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source, } bool has_gguf = false; + uint64_t model_size = 0; for (const auto& value : result.value()) { auto path = value["path"].asString(); if (path.find(".gguf") != std::string::npos) { has_gguf = true; + model_size = value["size"].asUInt64(); } } if (!has_gguf) { CTL_INF("Only support gguf file format! - branch: " << branch); return {}; } else { + auto meta_json = json_helper::ParseJsonString(metadata); + meta_json["size"] = model_size; + meta_json["description"] = desc; std::string model_id = model_name + ":" + branch; - cortex::db::ModelEntry e = {.model = model_id, - .author_repo_id = author, - .branch_name = branch, - .path_to_model_yaml = "", - .model_alias = "", - .model_format = "cortexso", - .model_source = model_source, - .status = cortex::db::ModelStatus::Downloadable, - .engine = "llama-cpp", - .metadata = metadata}; + cortex::db::ModelEntry e = { + .model = model_id, + .author_repo_id = author, + .branch_name = branch, + .path_to_model_yaml = "", + .model_alias = "", + .model_format = "cortexso", + .model_source = model_source, + .status = cortex::db::ModelStatus::Downloadable, + .engine = "llama-cpp", + .metadata = json_helper::DumpJsonString(meta_json)}; if (!db_service_->HasModel(model_id)) { CTL_INF("Adding model to db: " << model_name << ":" << branch); if (auto res = db_service_->AddModelEntry(e); @@ -426,37 +473,35 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source, } void ModelSourceService::SyncModelSource() { - // Do interval check for 10 minutes - constexpr const int kIntervalCheck = 10 * 60; - auto start_time = std::chrono::steady_clock::now(); while (running_) { std::this_thread::sleep_for(std::chrono::milliseconds(500)); - auto current_time = std::chrono::steady_clock::now(); - auto elapsed_time = std::chrono::duration_cast( - current_time - start_time) - .count(); - - if (elapsed_time > kIntervalCheck) { + auto now = std::chrono::system_clock::now(); + auto config = file_manager_utils::GetCortexConfig(); + auto last_check = + std::chrono::system_clock::time_point( + std::chrono::milliseconds(config.checkedForSyncHubAt)) + + std::chrono::hours(1); + + if (now > last_check) { CTL_DBG("Start to sync cortex.db"); - start_time = current_time; auto res = db_service_->GetModelSources(); if (res.has_error()) { CTL_INF(res.error()); } else { for (auto const& src : res.value()) { - CTL_DBG(src); + CTL_DBG(src.model_source); } std::unordered_set orgs; std::vector repos; for (auto const& src : res.value()) { - auto url_res = url_parser::FromUrlString(src); + auto url_res = url_parser::FromUrlString(src.model_source); if (url_res.has_value()) { if (url_res->pathParams.size() == 1) { - orgs.insert(src); + orgs.insert(src.model_source); } else if (url_res->pathParams.size() == 2) { - repos.push_back(src); + repos.push_back(src.model_source); } } } @@ -481,6 +526,20 @@ void ModelSourceService::SyncModelSource() { } CTL_DBG("Done sync cortex.db"); + + auto now = std::chrono::system_clock::now(); + auto config = file_manager_utils::GetCortexConfig(); + config.checkedForSyncHubAt = + std::chrono::duration_cast( + now.time_since_epoch()) + .count(); + + auto upd_config_res = + config_yaml_utils::CortexConfigMgr::GetInstance().DumpYamlConfig( + config, file_manager_utils::GetConfigurationPath().string()); + if (upd_config_res.has_error()) { + CTL_ERR("Failed to update config file: " << upd_config_res.error()); + } } } } diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h index 7227267d3..606d25d4f 100644 --- a/engine/services/model_source_service.h +++ b/engine/services/model_source_service.h @@ -1,10 +1,42 @@ #pragma once #include #include +#include #include #include "services/database_service.h" #include "utils/result.hpp" +struct ModelSourceInfo { + std::string id; + uint64_t size; + Json::Value ToJson() const { + Json::Value root; + root["id"] = id; + root["size"] = size; + return root; + } +}; + +struct ModelSource { + std::string id; + std::string author; + std::vector models; + Json::Value metadata; + + Json::Value ToJson() { + Json::Value root; + root["id"] = id; + root["author"] = author; + Json::Value models_json; + for (auto const& m : models) { + models_json.append(m.ToJson()); + } + root["models"] = models_json; + root["metadata"] = metadata; + return root; + }; +}; + class ModelSourceService { public: explicit ModelSourceService(std::shared_ptr db_service); @@ -16,7 +48,10 @@ class ModelSourceService { cpp::result RemoveModelSource( const std::string& model_source); - cpp::result, std::string> GetModelSources(); + cpp::result, std::string> + GetModelSources(); + + cpp::result GetModelSource(const std::string& src); private: cpp::result AddHfOrg(const std::string& model_source, @@ -41,7 +76,8 @@ class ModelSourceService { AddCortexsoRepoBranch(const std::string& model_source, const std::string& author, const std::string& model_name, - const std::string& branch, const std::string& metadata); + const std::string& branch, const std::string& metadata, + const std::string& desc); void SyncModelSource(); diff --git a/engine/utils/config_yaml_utils.cc b/engine/utils/config_yaml_utils.cc index 8fbfe1dbe..b26d690c6 100644 --- a/engine/utils/config_yaml_utils.cc +++ b/engine/utils/config_yaml_utils.cc @@ -50,6 +50,7 @@ cpp::result CortexConfigMgr::DumpYamlConfig( node["sslCertPath"] = config.sslCertPath; node["sslKeyPath"] = config.sslKeyPath; node["supportedEngines"] = config.supportedEngines; + node["checkedForSyncHubAt"] = config.checkedForSyncHubAt; out_file << node; out_file.close(); @@ -85,7 +86,8 @@ CortexConfig CortexConfigMgr::FromYaml(const std::string& path, !node["verifyPeerSsl"] || !node["verifyHostSsl"] || !node["verifyProxySsl"] || !node["verifyProxyHostSsl"] || !node["supportedEngines"] || !node["sslCertPath"] || - !node["sslKeyPath"] || !node["noProxy"]); + !node["sslKeyPath"] || !node["noProxy"] || + !node["checkedForSyncHubAt"]); CortexConfig config = { .logFolderPath = node["logFolderPath"] @@ -177,6 +179,9 @@ CortexConfig CortexConfigMgr::FromYaml(const std::string& path, node["supportedEngines"] ? node["supportedEngines"].as>() : default_cfg.supportedEngines, + .checkedForSyncHubAt = node["checkedForSyncHubAt"] + ? node["checkedForSyncHubAt"].as() + : default_cfg.checkedForSyncHubAt, }; if (should_update_config) { l.unlock(); diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h index 1f4b242ce..1749cd2d0 100644 --- a/engine/utils/config_yaml_utils.h +++ b/engine/utils/config_yaml_utils.h @@ -67,6 +67,7 @@ struct CortexConfig { std::string sslCertPath; std::string sslKeyPath; std::vector supportedEngines; + uint64_t checkedForSyncHubAt; }; class CortexConfigMgr { diff --git a/engine/utils/file_manager_utils.cc b/engine/utils/file_manager_utils.cc index a83c93efa..c04fef1e6 100644 --- a/engine/utils/file_manager_utils.cc +++ b/engine/utils/file_manager_utils.cc @@ -189,6 +189,7 @@ config_yaml_utils::CortexConfig GetDefaultConfig() { .sslCertPath = "", .sslKeyPath = "", .supportedEngines = config_yaml_utils::kDefaultSupportedEngines, + .checkedForSyncHubAt = 0u, }; } diff --git a/engine/utils/huggingface_utils.h b/engine/utils/huggingface_utils.h index 1d1040612..e5c74a6e1 100644 --- a/engine/utils/huggingface_utils.h +++ b/engine/utils/huggingface_utils.h @@ -22,6 +22,107 @@ struct HuggingFaceFileSibling { std::string rfilename; }; +struct HuggingFaceFileSize { + uint64_t size_in_bytes; +}; + +struct HuggingFaceSiblingsFileSize { + std::unordered_map file_sizes; + static cpp::result FromJson( + const Json::Value& json) { + if (json.isNull() || json.type() == Json::ValueType::nullValue) { + return cpp::fail("gguf info is null"); + } + + try { + HuggingFaceSiblingsFileSize res; + for (auto const& j : json) { + if (j["type"].asString() == "file") { + res.file_sizes[j["path"].asString()] = + HuggingFaceFileSize{.size_in_bytes = j["size"].asUInt64()}; + } + } + return res; + } catch (const std::exception& e) { + return cpp::fail("Failed to parse gguf info: " + std::string(e.what())); + } + } + + Json::Value ToJson() { + Json::Value root; + Json::Value siblings(Json::arrayValue); + for (auto const& s : file_sizes) { + Json::Value s_json; + s_json["path"] = s.first; + s_json["size"] = s.second.size_in_bytes; + siblings.append(s_json); + } + root["siblings"] = siblings; + return root; + } +}; + +inline cpp::result +GetSiblingsFileSize(const std::string& author, const std::string& model_name, + const std::string& branch = "main") { + if (author.empty() || model_name.empty()) { + return cpp::fail("Author and model name cannot be empty"); + } + auto url_obj = url_parser::Url{ + .protocol = "https", + .host = kHuggingFaceHost, + .pathParams = {"api", "models", author, model_name, "tree", branch}}; + + auto result = curl_utils::SimpleGetJson(url_obj.ToFullPath()); + if (result.has_error()) { + return cpp::fail("Failed to get model siblings file size: " + author + "/" + + model_name + "/tree/" + branch); + } + auto r = result.value(); + for (auto const& j : result.value()) { + if (j["type"].asString() == "directory") { + auto url_obj = + url_parser::Url{.protocol = "https", + .host = kHuggingFaceHost, + .pathParams = {"api", "models", author, model_name, + "tree", branch, j["path"].asString()}}; + + auto rd = curl_utils::SimpleGetJson(url_obj.ToFullPath()); + if (rd.has_value()) { + for (auto const& rdj : rd.value()) { + r.append(rdj); + } + } + } + } + + return HuggingFaceSiblingsFileSize::FromJson(r); +} + +inline cpp::result GetReadMe( + const std::string& author, const std::string& model_name) { + if (author.empty() || model_name.empty()) { + return cpp::fail("Author and model name cannot be empty"); + } + auto url_obj = url_parser::Url{.protocol = "https", + .host = kHuggingFaceHost, + .pathParams = { + author, + model_name, + "raw", + "main", + "README.md", + }}; + + auto result = curl_utils::SimpleGet(url_obj.ToFullPath()); + if (result.has_error()) { + return cpp::fail("Failed to get model siblings file size: " + author + "/" + + model_name + "/raw/main/README.md"); + } + + return result.value(); +} + struct HuggingFaceGgufInfo { uint64_t total; std::string architecture;