Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

feat: add filter compatible for engine variant api #1819

Merged
merged 1 commit into from
Dec 23, 2024
Merged

feat: add filter compatible for engine variant api #1819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions docs/static/openapi/cortex.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2199,6 +2199,84 @@
"tags": ["Engines"]
}
},
"/v1/engines/{name}/releases/{version}": {
"get": {
"summary": "List variants for a specific engine version",
"description": "Lists all available variants (builds) for a specific version of an engine. Variants can include different CPU architectures (AVX, AVX2, AVX512), GPU support (CUDA, Vulkan), and operating systems (Windows, Linux, macOS).",
"parameters": [
{
"name": "name",
"in": "path",
"required": true,
"schema": {
"type": "string",
"enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"],
"default": "llama-cpp"
},
"description": "The type of engine"
},
{
"name": "version",
"in": "path",
"required": true,
"schema": {
"type": "string"
},
"description": "The version of the engine"
},
{
"name": "show",
"in": "query",
"required": false,
"schema": {
"type": "string",
"enum": ["all", "compatible"],
"default": "all"
},
"description": "Filter the variants list. Use 'compatible' to show only variants compatible with the current system, or 'all' to show all available variants."
}
],
"responses": {
"200": {
"description": "Successfully retrieved variants list",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "The name of the variant, including OS, architecture, and capabilities",
"example": "linux-amd64-avx-cuda-11-7"
},
"created_at": {
"type": "string",
"format": "date-time",
"description": "Creation timestamp of the variant",
"example": "2024-11-13T04:51:16Z"
},
"size": {
"type": "integer",
"description": "Size of the variant in bytes",
"example": 151224604
},
"download_count": {
"type": "integer",
"description": "Number of times this variant has been downloaded",
"example": 0
}
}
}
}
}
}
}
},
"tags": ["Engines"]
}
},
"/v1/engines/{name}/releases/latest": {
"get": {
"summary": "Get latest release",
Expand Down
16 changes: 14 additions & 2 deletions engine/controllers/engines.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ void Engines::GetEngineReleases(
void Engines::GetEngineVariants(
const HttpRequestPtr& req,
std::function<void(const HttpResponsePtr&)>&& callback,
const std::string& engine, const std::string& version) const {
const std::string& engine, const std::string& version,
std::optional<std::string> show) const {
if (engine.empty()) {
Json::Value res;
res["message"] = "Engine name is required";
Expand All @@ -140,7 +141,18 @@ void Engines::GetEngineVariants(
return;
}

auto result = engine_service_->GetEngineVariants(engine, version);
auto show_value = show.value_or("all");
if (show_value != "all" && show_value != "compatible") {
Json::Value res;
res["message"] = "Invalid show value. Can either be `all` or `compatible`";
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k400BadRequest);
callback(resp);
return;
}

auto result = engine_service_->GetEngineVariants(engine, version,
show_value == "compatible");

auto normalize_version = string_utils::RemoveSubstring(version, "v");
Json::Value releases(Json::arrayValue);
Expand Down
12 changes: 5 additions & 7 deletions engine/controllers/engines.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,11 @@ class Engines : public drogon::HttpController<Engines, false> {
METHOD_ADD(Engines::GetEngineReleases, "/{1}/releases", Get);
ADD_METHOD_TO(Engines::GetEngineReleases, "/v1/engines/{1}/releases", Get);

METHOD_ADD(Engines::GetEngineVariants, "/{1}/releases/{2}", Get);
ADD_METHOD_TO(Engines::GetEngineVariants, "/v1/engines/{1}/releases/{2}",
Get);
ADD_METHOD_TO(Engines::GetEngineVariants,
"/v1/engines/{engine}/releases/{version}?show={show}", Get);

METHOD_ADD(Engines::GetLatestEngineVersion, "/{1}/releases/latest", Get);
ADD_METHOD_TO(Engines::GetLatestEngineVersion,
"/v1/engines/{1}/releases/latest", Get);
"/v1/engines/{engine}/releases/latest", Get);

METHOD_LIST_END

Expand All @@ -83,8 +81,8 @@ class Engines : public drogon::HttpController<Engines, false> {

void GetEngineVariants(const HttpRequestPtr& req,
std::function<void(const HttpResponsePtr&)>&& callback,
const std::string& engine,
const std::string& version) const;
const std::string& engine, const std::string& version,
std::optional<std::string> show) const;

void GetInstalledEngineVariants(
const HttpRequestPtr& req,
Expand Down
41 changes: 40 additions & 1 deletion engine/services/engine_service.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,8 @@ EngineService::GetEngineReleases(const std::string& engine) const {

cpp::result<std::vector<EngineService::EngineVariant>, std::string>
EngineService::GetEngineVariants(const std::string& engine,
const std::string& version) const {
const std::string& version,
bool filter_compatible_only) const {
auto ne = NormalizeEngine(engine);
auto engine_release =
github_release_utils::GetReleaseByVersion("janhq", ne, version);
Expand All @@ -506,6 +507,44 @@ EngineService::GetEngineVariants(const std::string& engine,
return cpp::fail("No compatible variants found for " + engine);
}

if (filter_compatible_only) {
auto system_info = system_info_utils::GetSystemInfo();
compatible_variants.erase(
std::remove_if(compatible_variants.begin(), compatible_variants.end(),
[&system_info](const EngineVariant& variant) {
std::string name = variant.name;
std::transform(name.begin(), name.end(), name.begin(),
::tolower);

bool os_match = false;
if (system_info->os == "mac" &&
name.find("mac") != std::string::npos)
os_match = true;
if (system_info->os == "windows" &&
name.find("windows") != std::string::npos)
os_match = true;
if (system_info->os == "linux" &&
name.find("linux") != std::string::npos)
os_match = true;

bool arch_match = false;
if (system_info->arch == "arm64" &&
name.find("arm64") != std::string::npos)
arch_match = true;
if (system_info->arch == "amd64" &&
name.find("amd64") != std::string::npos)
arch_match = true;

return !(os_match && arch_match);
}),
compatible_variants.end());

if (compatible_variants.empty()) {
return cpp::fail("No compatible variants found for system " +
system_info->os + "/" + system_info->arch);
}
}

return compatible_variants;
}

Expand Down
3 changes: 2 additions & 1 deletion engine/services/engine_service.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ class EngineService : public EngineServiceI {
const std::string& engine) const;

cpp::result<std::vector<EngineVariant>, std::string> GetEngineVariants(
const std::string& engine, const std::string& version) const;
const std::string& engine, const std::string& version,
bool filter_compatible_only = false) const;

cpp::result<DefaultEngineVariant, std::string> SetDefaultEngineVariant(
const std::string& engine, const std::string& version,
Expand Down
Loading