test

Author: namchuai

engine/services/inference_service.cc

@@ -28,8 +28,15 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
   }
 
   {
-    if (json_body->isMember("files") && !(*json_body)["files"].empty()) {
-      auto file = (*json_body)["files"][0].asString();
+    // TODO: we can cache this one so we don't have to read the file every inference
+    CTL_INF("Try to render chat_template");
+    auto model_id = json_body->get("model", "").asString();
+    if (!model_id.empty()) {
+      auto file =
+          "models/huggingface.co/bartowski/QwQ-32B-Preview-GGUF/"
+          "QwQ-32B-Preview-Q4_0.gguf";
+
+      // auto file = (*json_body)["files"][0].asString();
       auto model_metadata_res = cortex_utils::ReadGgufMetadata(
           file_manager_utils::ToAbsoluteCortexDataPath(
               std::filesystem::path(file)));
@@ -56,6 +63,8 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
             CTL_ERR("Failed to render prompt: " + prompt_result.error());
           }
         }
+      } else {
+        CTL_ERR("Failed to read metadata: " + model_metadata_res.error());
       }
     }
   }