Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

fix: app hangs if we unload model during stream completion #513

Merged
merged 3 commits into from
Apr 17, 2024
Merged

fix: app hangs if we unload model during stream completion #513

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
876790f
fix: e2e testing for linux and mac
vansangpfiev Apr 16, 2024
849f7b4
fix: thread hang if unload model when still receiving stream data
vansangpfiev Apr 16, 2024
9608a0c
Revert "fix: e2e testing for linux and mac"
vansangpfiev Apr 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions controllers/llamaCPP.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,7 @@ void llamaCPP::InferenceImpl(
if (llama.model_type == ModelType::EMBEDDING) {
LOG_WARN << "Not support completion for embedding model";
Json::Value jsonResp;
jsonResp["message"] =
"Not support completion for embedding model";
jsonResp["message"] = "Not support completion for embedding model";
auto resp = nitro_utils::nitroHttpJsonResponse(jsonResp);
resp->setStatusCode(drogon::k400BadRequest);
callback(resp);
Expand Down Expand Up @@ -429,7 +428,8 @@ void llamaCPP::InferenceImpl(

// Since this is an async task, we will wait for the task to be
// completed
while (state->inference_status != FINISHED && retries < 10) {
while (state->inference_status != FINISHED && retries < 10 &&
state->instance->llama.model_loaded_external) {
// Should wait chunked_content_provider lambda to be called within
// 3s
if (state->inference_status == PENDING) {
Expand Down Expand Up @@ -748,9 +748,10 @@ void llamaCPP::StopBackgroundTask() {
if (llama.model_loaded_external) {
llama.model_loaded_external = false;
llama.condition_tasks.notify_one();
LOG_INFO << "Background task stopped! ";
LOG_INFO << "Stopping background task! ";
if (backgroundThread.joinable()) {
backgroundThread.join();
}
LOG_INFO << "Background task stopped! ";
}
}