From c1817e66189dd069358350188706122110863462 Mon Sep 17 00:00:00 2001 From: tikikun Date: Thu, 9 Nov 2023 16:27:50 +0700 Subject: [PATCH 1/2] fix parameter issues and add pre prompt --- controllers/llamaCPP.cc | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/controllers/llamaCPP.cc b/controllers/llamaCPP.cc index 24b5c9718..868b7aa06 100644 --- a/controllers/llamaCPP.cc +++ b/controllers/llamaCPP.cc @@ -76,22 +76,20 @@ void llamaCPP::chatCompletion( std::function &&callback) { const auto &jsonBody = req->getJsonObject(); - std::string formatted_output = - "Below is a conversation between an AI system named " + ai_prompt + - " and " + user_prompt + "\n"; + std::string formatted_output = pre_prompt; json data; json stopWords; // To set default value data["stream"] = true; - data["n_predict"] = 30; if (jsonBody) { - data["n_predict"] = (*jsonBody)["max_tokens"].asInt(); - data["top_p"] = (*jsonBody)["top_p"].asFloat(); - data["temperature"] = (*jsonBody)["temperature"].asFloat(); - data["frequency_penalty"] = (*jsonBody)["frequency_penalty"].asFloat(); - data["presence_penalty"] = (*jsonBody)["presence_penalty"].asFloat(); + data["n_predict"] = (*jsonBody).get("max_tokens", 500).asInt(); + data["top_p"] = (*jsonBody).get("top_p", 0.95).asFloat(); + data["temperature"] = (*jsonBody).get("temperature", 0.8).asFloat(); + data["frequency_penalty"] = + (*jsonBody).get("frequency_penalty", 0).asFloat(); + data["presence_penalty"] = (*jsonBody).get("presence_penalty", 0).asFloat(); const Json::Value &messages = (*jsonBody)["messages"]; for (const auto &message : messages) { @@ -109,7 +107,7 @@ void llamaCPP::chatCompletion( std::string content = message["content"].asString(); formatted_output += role + content + "\n"; } - formatted_output += "assistant:"; + formatted_output += ai_prompt; data["prompt"] = formatted_output; for (const auto &stop_word : (*jsonBody)["stop"]) { @@ -225,6 +223,13 @@ void llamaCPP::loadModel( this->ai_prompt = (*jsonBody).get("ai_prompt", "ASSISTANT: ").asString(); this->system_prompt = (*jsonBody).get("system_prompt", "ASSISTANT's RULE: ").asString(); + this->pre_prompt = + (*jsonBody) + .get("pre_prompt", + "A chat between a curious user and an artificial intelligence " + "assistant. The assistant follows the given rules no matter " + "what.\\n") + .asString(); } #ifdef GGML_USE_CUBLAS LOG_INFO << "Setting up GGML CUBLAS PARAMS"; From 1ec17e99e5f8bd812cd52b1ea5ba4f1c44ca822f Mon Sep 17 00:00:00 2001 From: tikikun Date: Thu, 9 Nov 2023 16:28:23 +0700 Subject: [PATCH 2/2] add pre prompt --- controllers/llamaCPP.h | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/llamaCPP.h b/controllers/llamaCPP.h index 037cae926..4e755d524 100644 --- a/controllers/llamaCPP.h +++ b/controllers/llamaCPP.h @@ -2145,5 +2145,6 @@ class llamaCPP : public drogon::HttpController { std::string user_prompt; std::string ai_prompt; std::string system_prompt; + std::string pre_prompt; }; }; // namespace inferences