Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

fix: model update command only accept string type #1510

Merged
merged 2 commits into from
Oct 17, 2024
Merged

fix: model update command only accept string type #1510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
442eefb
fix: revert check type for json on model update command
sangjanai Oct 17, 2024
84c49ef
fix: build
sangjanai Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
288 changes: 285 additions & 3 deletions engine/commands/model_upd_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#include "model_upd_cmd.h"
#include "httplib.h"
#include "json/json.h"

#include "server_start_cmd.h"
#include "utils/file_manager_utils.h"
#include "utils/logging_utils.h"
Expand All @@ -26,8 +26,7 @@ void ModelUpdCmd::Exec(
Json::Value json_data;
for (const auto& [key, value] : options) {
if (!value.empty()) {
json_data[key] = value;
CLI_LOG("Updated " << key << " to: " << value);
UpdateConfig(json_data, key, value);
}
}
auto data_str = json_data.toStyledString();
Expand All @@ -47,4 +46,287 @@ void ModelUpdCmd::Exec(
return;
}
}

void ModelUpdCmd::UpdateConfig(Json::Value& data, const std::string& key,
const std::string& value) {
static const std::unordered_map<
std::string,
std::function<void(Json::Value &, const std::string&, const std::string&)>>
updaters = {
{"name",
[](Json::Value &data, const std::string&, const std::string& v) {
data["name"] = v;
}},
{"model",
[](Json::Value &data, const std::string&, const std::string& v) {
data["model"] = v;
}},
{"version",
[](Json::Value &data, const std::string&, const std::string& v) {
data["version"] = v;
}},
{"engine",
[](Json::Value &data, const std::string&, const std::string& v) {
data["engine"] = v;
}},
{"prompt_template",
[](Json::Value &data, const std::string&, const std::string& v) {
data["prompt_template"] = v;
}},
{"system_template",
[](Json::Value &data, const std::string&, const std::string& v) {
data["system_template"] = v;
}},
{"user_template",
[](Json::Value &data, const std::string&, const std::string& v) {
data["user_template"] = v;
}},
{"ai_template",
[](Json::Value &data, const std::string&, const std::string& v) {
data["ai_template"] = v;
}},
{"os",
[](Json::Value &data, const std::string&, const std::string& v) {
data["os"] = v;
}},
{"gpu_arch",
[](Json::Value &data, const std::string&, const std::string& v) {
data["gpu_arch"] = v;
}},
{"quantization_method",
[](Json::Value &data, const std::string&, const std::string& v) {
data["quantization_method"] = v;
}},
{"precision",
[](Json::Value &data, const std::string&, const std::string& v) {
data["precision"] = v;
}},
{"trtllm_version",
[](Json::Value &data, const std::string&, const std::string& v) {
data["trtllm_version"] = v;
}},
{"object",
[](Json::Value &data, const std::string&, const std::string& v) {
data["object"] = v;
}},
{"owned_by",
[](Json::Value &data, const std::string&, const std::string& v) {
data["owned_by"] = v;
}},
{"grammar",
[](Json::Value &data, const std::string&, const std::string& v) {
data["grammar"] = v;
}},
{"stop", [this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateVectorField(
k, v, [&data](const std::vector<std::string>& stops) {
Json::Value d(Json::arrayValue);
for (auto const& s: stops) {
d.append(s);
}
data["stop"] = d;
});
}},
{"files", [this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateVectorField(
k, v, [&data](const std::vector<std::string>& fs) {
Json::Value d(Json::arrayValue);
for (auto const& f: fs) {
d.append(f);
}
data["files"] = d;
});
}},
{"top_p",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(
k, v, [&data](float f) { data["top_p"] = f; });
}},
{"temperature",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["temperature"] = f;
});
}},
{"frequency_penalty",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["frequency_penalty"] = f;
});
}},
{"presence_penalty",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["presence_penalty"] = f;
});
}},
{"dynatemp_range",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["dynatemp_range"] = f;
});
}},
{"dynatemp_exponent",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["dynatemp_exponent"] = f;
});
}},
{"min_p",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(
k, v, [&data](float f) { data["min_p"] = f; });
}},
{"tfs_z",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(
k, v, [&data](float f) { data["tfs_z"] = f; });
}},
{"typ_p",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(
k, v, [&data](float f) { data["typ_p"] = f; });
}},
{"repeat_penalty",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["repeat_penalty"] = f;
});
}},
{"mirostat_tau",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["mirostat_tau"] = f;
});
}},
{"mirostat_eta",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["mirostat_eta"] = f;
});
}},
{"max_tokens",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["max_tokens"] = static_cast<int>(f);
});
}},
{"ngl",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["ngl"] = static_cast<int>(f);
});
}},
{"ctx_len",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["ctx_len"] = static_cast<int>(f);
});
}},
{"tp",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["tp"] = static_cast<int>(f);
});
}},
{"seed",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["seed"] = static_cast<int>(f);
});
}},
{"top_k",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["top_k"] = static_cast<int>(f);
});
}},
{"repeat_last_n",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["repeat_last_n"] = static_cast<int>(f);
});
}},
{"n_probs",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["n_probs"] = static_cast<int>(f);
});
}},
{"min_keep",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["min_keep"] = static_cast<int>(f);
});
}},
{"stream",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateBooleanField(
k, v, [&data](bool b) { data["stream"] = b; });
}},
{"text_model",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateBooleanField(
k, v, [&data](bool b) { data["text_model"] = b; });
}},
{"mirostat",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateBooleanField(
k, v, [&data](bool b) { data["mirostat"] = b; });
}},
{"penalize_nl",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateBooleanField(
k, v, [&data](bool b) { data["penalize_nl"] = b; });
}},
{"ignore_eos",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateBooleanField(
k, v, [&data](bool b) { data["ignore_eos"] = b; });
}},
{"created",
[this](Json::Value &data, const std::string& k, const std::string& v) {
UpdateNumericField(k, v, [&data](float f) {
data["created"] = static_cast<uint64_t>(f);
});
}},
};

if (auto it = updaters.find(key); it != updaters.end()) {
it->second(data, key, value);
CLI_LOG("Updated " << key << " to: " << value);
} else {
CLI_LOG("Warning: Unknown configuration key '" << key << "' ignored.");
}
}

void ModelUpdCmd::UpdateVectorField(
const std::string& key, const std::string& value,
std::function<void(const std::vector<std::string>&)> setter) {
std::vector<std::string> tokens;
std::istringstream iss(value);
std::string token;
while (std::getline(iss, token, ',')) {
tokens.push_back(token);
}
setter(tokens);
}

void ModelUpdCmd::UpdateNumericField(const std::string& key,
const std::string& value,
std::function<void(float)> setter) {
try {
float numericValue = std::stof(value);
setter(numericValue);
} catch (const std::exception& e) {
CLI_LOG("Failed to parse numeric value for " << key << ": " << e.what());
}
}

void ModelUpdCmd::UpdateBooleanField(const std::string& key,
const std::string& value,
std::function<void(bool)> setter) {
bool boolValue = (value == "true" || value == "1");
setter(boolValue);
}
} // namespace commands
14 changes: 14 additions & 0 deletions engine/commands/model_upd_cmd.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,27 @@
#include <string>
#include <unordered_map>
#include <vector>
#include <functional>
#include "json/json.h"
namespace commands {
class ModelUpdCmd {
public:
ModelUpdCmd(std::string model_handle);

void Exec(const std::string& host, int port,
const std::unordered_map<std::string, std::string>& options);

private:
void UpdateConfig(Json::Value& data, const std::string& key,
const std::string& value);
void UpdateVectorField(
const std::string& key, const std::string& value,
std::function<void(const std::vector<std::string>&)> setter);
void UpdateNumericField(const std::string& key, const std::string& value,
std::function<void(float)> setter);
void UpdateBooleanField(const std::string& key, const std::string& value,
std::function<void(bool)> setter);

private:
std::string model_handle_;
};
Expand Down
Loading