diff --git a/engine/extensions/local-engine/local_engine.cc b/engine/extensions/local-engine/local_engine.cc index 885c14d77..b769c5e8c 100644 --- a/engine/extensions/local-engine/local_engine.cc +++ b/engine/extensions/local-engine/local_engine.cc @@ -20,7 +20,7 @@ const std::unordered_set kIgnoredParams = { "user_prompt", "min_keep", "mirostat", "mirostat_eta", "mirostat_tau", "text_model", "version", "n_probs", "object", "penalize_nl", "precision", "size", - "stop", "tfs_z", "typ_p"}; + "stop", "tfs_z", "typ_p", "caching_enabled"}; const std::unordered_map kParamsMap = { {"cpu_threads", "--threads"}, @@ -67,6 +67,19 @@ std::vector ConvertJsonToParamsVector(const Json::Value& root) { res.push_back("--embedding"); } continue; + } else if (member == "cache_type") { + if (!root[member].isNull()) { + res.push_back("-ctk"); + res.push_back(root[member].asString()); + res.push_back("-ctv"); + res.push_back(root[member].asString()); + } + continue; + } else if (member == "use_mmap") { + if (!root[member].asBool()) { + res.push_back("--no-mmap"); + } + continue; } res.push_back("--" + member); diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc index 89cd00058..15c7148c7 100644 --- a/engine/services/engine_service.cc +++ b/engine/services/engine_service.cc @@ -772,7 +772,13 @@ EngineService::GetInstalledEngineVariants(const std::string& engine) const { // try to find version.txt auto version_txt_path = version_entry.path() / "version.txt"; if (!std::filesystem::exists(version_txt_path)) { - continue; + // create new one + std::ofstream meta(version_txt_path, std::ios::out); + meta << "name: " << entry.path().filename() << std::endl; + meta << "version: " << version_entry.path().filename() << std::endl; + meta.close(); + CTL_INF("name: " << entry.path().filename().string() << ", version: " + << version_entry.path().filename().string()); } try { @@ -865,7 +871,9 @@ void EngineService::RegisterEngineLibPath() { // register deps std::vector paths{}; - paths.push_back(cuda_path); + if (std::filesystem::exists(cuda_path)) { + paths.push_back(cuda_path); + } paths.push_back(engine_dir_path); CTL_DBG("Registering dylib for "