Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit bd71837

Browse files
nguyenhoangthuan99nguyenhoangthuan99
committed
models get command
1 parent 1b2e092 commit bd71837

File tree

6 files changed

+224
-1
lines changed

6 files changed

+224
-1
lines changed

‎engine/.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -563,4 +563,5 @@ build
563563
build-deps
564564
.DS_Store
565565

566-
uploads/**
566+
uploads/**
567+
CMakePresets.json

‎engine/commands/model_get_cmd.cc‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#include "model_get_cmd.h"
2+
#include <filesystem>
3+
#include <iostream>
4+
#include <vector>
5+
#include "config/yaml_config.h"
6+
#include "trantor/utils/Logger.h"
7+
#include "utils/cortex_utils.h"
8+
9+
namespace commands {
10+
ModelGetCmd::ModelGetCmd(std::string modelHandle)
11+
: modelHandle_(std::move(modelHandle)) {}
12+
13+
void ModelGetCmd::Exec() {
14+
if (std::filesystem::exists(cortex_utils::models_folder) &&
15+
std::filesystem::is_directory(cortex_utils::models_folder)) {
16+
bool found_model = false;
17+
// Iterate through directory
18+
for (const auto& entry :
19+
std::filesystem::directory_iterator(cortex_utils::models_folder)) {
20+
if (entry.is_regular_file() && entry.path().stem() == modelHandle_ && entry.path().extension() == ".yaml") {
21+
try {
22+
config::YamlHandler handler;
23+
handler.ModelConfigFromFile(entry.path().string());
24+
const auto& model_config = handler.GetModelConfig();
25+
std::cout << "ModelConfig Details:\n";
26+
std::cout << "-------------------\n";
27+
28+
// Print non-null strings
29+
if (!model_config.id.empty())
30+
std::cout << "id: " << model_config.id << "\n";
31+
if (!model_config.name.empty())
32+
std::cout << "name: " << model_config.name << "\n";
33+
if (!model_config.model.empty())
34+
std::cout << "model: " << model_config.model << "\n";
35+
if (!model_config.version.empty())
36+
std::cout << "version: " << model_config.version << "\n";
37+
38+
// Print non-empty vectors
39+
if (!model_config.stop.empty()) {
40+
std::cout << "stop: [";
41+
for (size_t i = 0; i < model_config.stop.size(); ++i) {
42+
std::cout << model_config.stop[i];
43+
if (i < model_config.stop.size() - 1)
44+
std::cout << ", ";
45+
}
46+
std::cout << "]\n";
47+
}
48+
// Print valid numbers
49+
if (!std::isnan(static_cast<double>(model_config.top_p)))
50+
std::cout << "top_p: " << model_config.top_p << "\n";
51+
if (!std::isnan(static_cast<double>(model_config.temperature)))
52+
std::cout << "temperature: " << model_config.temperature << "\n";
53+
if (!std::isnan(static_cast<double>(model_config.frequency_penalty)))
54+
std::cout << "frequency_penalty: " << model_config.frequency_penalty
55+
<< "\n";
56+
if (!std::isnan(static_cast<double>(model_config.presence_penalty)))
57+
std::cout << "presence_penalty: " << model_config.presence_penalty
58+
<< "\n";
59+
if (!std::isnan(static_cast<double>(model_config.max_tokens)))
60+
std::cout << "max_tokens: " << model_config.max_tokens << "\n";
61+
if (!std::isnan(static_cast<double>(model_config.stream)))
62+
std::cout << "stream: " << std::boolalpha << model_config.stream << "\n";
63+
if (!std::isnan(static_cast<double>(model_config.ngl)))
64+
std::cout << "ngl: " << model_config.ngl << "\n";
65+
if (!std::isnan(static_cast<double>(model_config.ctx_len)))
66+
std::cout << "ctx_len: " << model_config.ctx_len << "\n";
67+
68+
// Print non-null strings
69+
if (!model_config.engine.empty())
70+
std::cout << "engine: " << model_config.engine << "\n";
71+
if (!model_config.prompt_template.empty())
72+
std::cout << "prompt_template: " << model_config.prompt_template << "\n";
73+
if (!model_config.system_template.empty())
74+
std::cout << "system_template: " << model_config.system_template << "\n";
75+
if (!model_config.user_template.empty())
76+
std::cout << "user_template: " << model_config.user_template << "\n";
77+
if (!model_config.ai_template.empty())
78+
std::cout << "ai_template: " << model_config.ai_template << "\n";
79+
if (!model_config.os.empty())
80+
std::cout << "os: " << model_config.os << "\n";
81+
if (!model_config.gpu_arch.empty())
82+
std::cout << "gpu_arch: " << model_config.gpu_arch << "\n";
83+
if (!model_config.quantization_method.empty())
84+
std::cout << "quantization_method: " << model_config.quantization_method
85+
<< "\n";
86+
if (!model_config.precision.empty())
87+
std::cout << "precision: " << model_config.precision << "\n";
88+
89+
if (!std::isnan(static_cast<double>(model_config.tp)))
90+
std::cout << "tp: " << model_config.tp << "\n";
91+
92+
// Print non-null strings
93+
if (!model_config.trtllm_version.empty())
94+
std::cout << "trtllm_version: " << model_config.trtllm_version << "\n";
95+
if (!std::isnan(static_cast<double>(model_config.text_model)))
96+
std::cout << "text_model: " << std::boolalpha << model_config.text_model
97+
<< "\n";
98+
99+
// Print non-empty vectors
100+
if (!model_config.files.empty()) {
101+
std::cout << "files: [";
102+
for (size_t i = 0; i < model_config.files.size(); ++i) {
103+
std::cout << model_config.files[i];
104+
if (i < model_config.files.size() - 1)
105+
std::cout << ", ";
106+
}
107+
std::cout << "]\n";
108+
}
109+
110+
// Print valid size_t number
111+
if (model_config.created != 0)
112+
std::cout << "created: " << model_config.created << "\n";
113+
114+
if (!model_config.object.empty())
115+
std::cout << "object: " << model_config.object << "\n";
116+
if (!model_config.owned_by.empty())
117+
std::cout << "owned_by: " << model_config.owned_by << "\n";
118+
119+
found_model = true;
120+
break;
121+
} catch (const std::exception& e) {
122+
LOG_ERROR << "Error reading yaml file '" << entry.path().string()
123+
<< "': " << e.what();
124+
}
125+
}
126+
}
127+
}
128+
}
129+
}; // namespace commands

‎engine/commands/model_get_cmd.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#pragma once
2+
3+
#include <string>
4+
#include <cmath> // For std::isnan
5+
namespace commands {
6+
7+
class ModelGetCmd {
8+
public:
9+
ModelGetCmd(std::string modelHandle);
10+
void Exec();
11+
12+
private:
13+
std::string modelHandle_;
14+
};
15+
} // namespace commands

‎engine/controllers/command_line_parser.cc‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include "commands/engine_init_cmd.h"
33
#include "commands/model_pull_cmd.h"
44
#include "commands/model_list_cmd.h"
5+
#include "commands/model_get_cmd.h"
56
#include "commands/start_model_cmd.h"
67
#include "commands/stop_model_cmd.h"
78
#include "commands/stop_server_cmd.h"
@@ -49,6 +50,14 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
4950
command.Exec();
5051
});
5152

53+
auto get_models_cmd =
54+
models_cmd->add_subcommand("get", "Get info of {model_id} locally");
55+
get_models_cmd->add_option("model_id", model_id, "");
56+
get_models_cmd->callback([&model_id](){
57+
commands::ModelGetCmd command(model_id);
58+
command.Exec();
59+
});
60+
5261
auto model_pull_cmd =
5362
app_.add_subcommand("pull",
5463
"Download a model from a registry. Working with "

‎engine/controllers/models.cc‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,4 +101,70 @@ void Models::ListModel(
101101
auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
102102
resp->setStatusCode(k200OK);
103103
callback(resp);
104+
}
105+
106+
void Models::GetModel(
107+
const HttpRequestPtr& req,
108+
std::function<void(const HttpResponsePtr&)>&& callback) const {
109+
if (!http_util::HasFieldInReq(req, callback, "modelId")) {
110+
return;
111+
}
112+
auto model_handle = (*(req->getJsonObject())).get("modelId", "").asString();
113+
LOG_DEBUG << "GetModel, Model handle: " << model_handle;
114+
Json::Value ret;
115+
ret["object"] = "list";
116+
Json::Value data(Json::arrayValue);
117+
if (std::filesystem::exists(cortex_utils::models_folder) &&
118+
std::filesystem::is_directory(cortex_utils::models_folder)) {
119+
// Iterate through directory
120+
for (const auto& entry :
121+
std::filesystem::directory_iterator(cortex_utils::models_folder)) {
122+
if (entry.is_regular_file() && entry.path().extension() == ".yaml" &&
123+
entry.path().stem() == model_handle) {
124+
try {
125+
config::YamlHandler handler;
126+
handler.ModelConfigFromFile(entry.path().string());
127+
auto const& model_config = handler.GetModelConfig();
128+
Json::Value obj;
129+
obj["name"] = model_config.name;
130+
obj["model"] = model_config.model;
131+
obj["version"] = model_config.version;
132+
Json::Value stop_array(Json::arrayValue);
133+
for (const std::string& stop : model_config.stop)
134+
stop_array.append(stop);
135+
obj["stop"] = stop_array;
136+
obj["top_p"] = model_config.top_p;
137+
obj["temperature"] = model_config.temperature;
138+
obj["presence_penalty"] = model_config.presence_penalty;
139+
obj["max_tokens"] = model_config.max_tokens;
140+
obj["stream"] = model_config.stream;
141+
obj["ngl"] = model_config.ngl;
142+
obj["ctx_len"] = model_config.ctx_len;
143+
obj["engine"] = model_config.engine;
144+
obj["prompt_template"] = model_config.prompt_template;
145+
146+
Json::Value files_array(Json::arrayValue);
147+
for (const std::string& file : model_config.files)
148+
files_array.append(file);
149+
obj["files"] = files_array;
150+
obj["id"] = model_config.id;
151+
obj["created"] = static_cast<uint32_t>(model_config.created);
152+
obj["object"] = model_config.object;
153+
obj["owned_by"] = model_config.owned_by;
154+
if (model_config.engine == "cortex.tensorrt-llm") {
155+
obj["trtllm_version"] = model_config.trtllm_version;
156+
}
157+
data.append(std::move(obj));
158+
} catch (const std::exception& e) {
159+
LOG_ERROR << "Error reading yaml file '" << entry.path().string()
160+
<< "': " << e.what();
161+
}
162+
}
163+
}
164+
}
165+
ret["data"] = data;
166+
ret["result"] = "OK";
167+
auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
168+
resp->setStatusCode(k200OK);
169+
callback(resp);
104170
}

‎engine/controllers/models.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,13 @@ class Models : public drogon::HttpController<Models> {
1414
METHOD_LIST_BEGIN
1515
METHOD_ADD(Models::PullModel, "/pull", Post);
1616
METHOD_ADD(Models::ListModel, "/list", Get);
17+
METHOD_ADD(Models::GetModel, "/get", Post);
1718
METHOD_LIST_END
1819

1920
void PullModel(const HttpRequestPtr& req,
2021
std::function<void(const HttpResponsePtr&)>&& callback) const;
2122
void ListModel(const HttpRequestPtr& req,
2223
std::function<void(const HttpResponsePtr&)>&& callback) const;
24+
void GetModel(const HttpRequestPtr& req,
25+
std::function<void(const HttpResponsePtr&)>&& callback) const;
2326
};

0 commit comments

Comments
 (0)