feat: support unload engine (#989)

Co-authored-by: vansangpfiev <sang@jan.ai>

Author: vansangpfiev

cortex-cpp/controllers/server.cc

@@ -19,12 +19,12 @@ constexpr static auto kOnnxEngine = "cortex.onnx";
 constexpr static auto kTensorrtLlmEngine = "cortex.tensorrt-llm";
 }  // namespace
 
-server::server(){
+server::server() {
 
-    // Some default values for now below
-    // log_disable();  // Disable the log to file feature, reduce bloat for
-    // target
-    // system ()
+  // Some default values for now below
+  // log_disable();  // Disable the log to file feature, reduce bloat for
+  // target
+  // system ()
 };
 
 server::~server() {}
@@ -326,6 +326,36 @@ void server::LoadModel(const HttpRequestPtr& req,
   LOG_TRACE << "Done load model";
 }
 
+void server::UnloadEngine(
+    const HttpRequestPtr& req,
+    std::function<void(const HttpResponsePtr&)>&& callback) {
+  if (!HasFieldInReq(req, callback, "engine")) {
+    return;
+  }
+
+  auto engine_type =
+      (*(req->getJsonObject())).get("engine", cur_engine_type_).asString();
+  if (!IsEngineLoaded(engine_type)) {
+    Json::Value res;
+    res["message"] = "Engine is not loaded yet";
+    auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
+    resp->setStatusCode(k409Conflict);
+    callback(resp);
+    LOG_WARN << "Engine is not loaded yet";
+    return;
+  }
+
+  EngineI* e = std::get<EngineI*>(engines_[engine_type].engine);
+  delete e;
+  engines_.erase(engine_type);
+  LOG_INFO << "Unloaded engine " + engine_type;
+  Json::Value res;
+  res["message"] = "Unloaded engine " + engine_type;
+  auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
+  resp->setStatusCode(k200OK);
+  callback(resp);
+}
+
 void server::ProcessStreamRes(std::function<void(const HttpResponsePtr&)> cb,
                               std::shared_ptr<SyncQueue> q) {
   auto err_or_done = std::make_shared<std::atomic_bool>(false);

cortex-cpp/controllers/server.h

@@ -66,6 +66,8 @@ class server : public drogon::HttpController<server>,
   // ADD_METHOD_TO(server::handlePrelight, "/v1/embeddings", Options);
 
   // PATH_ADD("/llama/chat_completion", Post);
+  METHOD_ADD(server::UnloadEngine, "unloadengine", Post);
+
   METHOD_LIST_END
   void ChatCompletion(
       const HttpRequestPtr& req,
@@ -91,6 +93,8 @@ class server : public drogon::HttpController<server>,
   void FineTuning(
       const HttpRequestPtr& req,
       std::function<void(const HttpResponsePtr&)>&& callback) override;
+  void UnloadEngine(const HttpRequestPtr& req,
+                    std::function<void(const HttpResponsePtr&)>&& callback);
 
  private:
   void ProcessStreamRes(std::function<void(const HttpResponsePtr&)> cb,