Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

hotfix: will always be cleaning cache #509

Merged
merged 5 commits into from
Apr 15, 2024
Merged

hotfix: will always be cleaning cache #509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d1133c1
will always be cleaning cache
tikikun Apr 15, 2024
91792dd
turn off caching
tikikun Apr 15, 2024
9a46624
force no caching
tikikun Apr 15, 2024
e2bc67a
hotfix: turn off current cache logic
tikikun Apr 15, 2024
1276fbc
turn off all kidns of caching
tikikun Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion context/llama_server_context.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@ struct llama_server_context {
llama_batch batch;

bool multimodal = false;
bool clean_kv_cache = false;
bool clean_kv_cache = true;
bool all_slots_are_idle = false;
bool add_bos_token = true;

Expand Down
17 changes: 9 additions & 8 deletions controllers/llamaCPP.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,15 +197,16 @@ void llamaCPP::InferenceImpl(
// To set default value

// Increase number of chats received and clean the prompt
no_of_chats++;
if (no_of_chats % clean_cache_threshold == 0) {
LOG_INFO_REQUEST(request_id) << "Clean cache threshold reached!";
llama.kv_cache_clear();
LOG_INFO_REQUEST(request_id) << "Cache cleaned";
}
//no_of_chats++;
//if (no_of_chats % clean_cache_threshold == 0) {
// LOG_INFO_REQUEST(request_id) << "Clean cache threshold reached!";
// llama.kv_cache_clear();
// LOG_INFO_REQUEST(request_id) << "Cache cleaned";
//}

// Default values to enable auto caching
data["cache_prompt"] = caching_enabled;
//data["cache_prompt"] = caching_enabled;
data["cache_prompt"] = false;
data["n_keep"] = -1;

// Passing load value
Expand Down Expand Up @@ -655,7 +656,7 @@ bool llamaCPP::LoadModelImpl(std::shared_ptr<Json::Value> jsonBody) {
params.cont_batching = jsonBody->get("cont_batching", false).asBool();
this->clean_cache_threshold =
jsonBody->get("clean_cache_threshold", 5).asInt();
this->caching_enabled = jsonBody->get("caching_enabled", true).asBool();
this->caching_enabled = jsonBody->get("caching_enabled", false).asBool();
this->user_prompt = jsonBody->get("user_prompt", "USER: ").asString();
this->ai_prompt = jsonBody->get("ai_prompt", "ASSISTANT: ").asString();
this->system_prompt =
Expand Down