Merge branch 'readme' of https://github.com/janhq/cortex.cpp into readme

Author: irfanpena

.github/workflows/cortex-cpp-quality-gate.yml

@@ -6,6 +6,7 @@ on:
     paths:
       [
         "engine/**",
+        ".github/workflows/cortex-cpp-quality-gate.yml"
       ]
   workflow_dispatch:
 
@@ -24,19 +25,19 @@ jobs:
           - os: "linux"
             name: "amd64"
             runs-on: "ubuntu-20-04-cuda-12-0"
-            cmake-flags: "-DCORTEX_CPP_VERSION=${{github.event.pull_request.head.sha}} -DCMAKE_BUILD_TEST=ON -DCMAKE_TOOLCHAIN_FILE=/home/runner/actions-runner/_work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake"
+            cmake-flags: "-DCORTEX_CPP_VERSION=${{github.event.pull_request.head.sha}} -DCMAKE_BUILD_TEST=ON -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake"
             build-deps-cmake-flags: ""
             ccache-dir: ''
           - os: "mac"
             name: "amd64"
-            runs-on: "macos-13"
-            cmake-flags: "-DCORTEX_CPP_VERSION=${{github.event.pull_request.head.sha}} -DCMAKE_BUILD_TEST=ON -DCMAKE_TOOLCHAIN_FILE=/Users/runner/work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake"
+            runs-on: "macos-selfhosted-12"
+            cmake-flags: "-DCORTEX_CPP_VERSION=${{github.event.pull_request.head.sha}} -DCMAKE_BUILD_TEST=ON -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake"
             build-deps-cmake-flags: ""
             ccache-dir: ''
           - os: "mac"
             name: "arm64"
-            runs-on: "macos-latest"
-            cmake-flags: "-DCORTEX_CPP_VERSION=${{github.event.pull_request.head.sha}} -DCMAKE_BUILD_TEST=ON -DMAC_ARM64=ON -DCMAKE_TOOLCHAIN_FILE=/Users/runner/work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake"
+            runs-on: "macos-silicon"
+            cmake-flags: "-DCORTEX_CPP_VERSION=${{github.event.pull_request.head.sha}} -DCMAKE_BUILD_TEST=ON -DMAC_ARM64=ON -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake"
             build-deps-cmake-flags: ""
             ccache-dir: ''
           - os: "windows"
@@ -51,7 +52,18 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: recursive
-      
+
+      - name: use python for linux
+        if: runner.os == 'Linux'
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install tools on Linux
+        if: runner.os == 'Linux'
+        run: |
+          python3 -m pip install awscli
+
       - name: Install choco on Windows
         if: runner.os == 'Windows'
         run: |
@@ -64,6 +76,17 @@ jobs:
           Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
           refreshenv
           aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-cpp-${{ matrix.os }}-${{ matrix.name }} ${{ matrix.ccache-dir }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
+          aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-cpp-vcpkg-windows C:\Users\ContainerAdministrator\AppData\Local\vcpkg --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
+          AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
+
+      - name: Download vcpkg cache from s3
+        if: runner.os == 'Linux'
+        continue-on-error: true
+        run: |
+          aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-cpp-vcpkg-linux /home/runner/.cache/vcpkg --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }} --cli-read-timeout 0
         env:
           AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
           AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
@@ -73,7 +96,7 @@ jobs:
         run: |
           cd engine
           make configure-vcpkg
-      
+
       - name: Build
         run: |
           cd engine
@@ -100,13 +123,24 @@ jobs:
           name: cortex-${{ matrix.os }}-${{ matrix.name }}
           path: ./engine/cortex
 
-      - name: Upload ccache to s3
+      - name: Upload windows ccache to s3
         continue-on-error: true
         if: always() && runner.os == 'Windows'
         run: |
           Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
           refreshenv
           aws s3 cp ${{ matrix.ccache-dir }} s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-${{ matrix.os }}-${{ matrix.name }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
+          aws s3 cp C:\Users\ContainerAdministrator\AppData\Local\vcpkg s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-cpp-vcpkg-windows --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
+          AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
+
+      - name: Upload linux vcpkg cache to s3
+        continue-on-error: true
+        if: always() && runner.os == 'Linux'
+        run: |
+          aws s3 cp /home/runner/.cache/vcpkg s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-cpp-vcpkg-linux --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
         env:
           AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
           AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"

engine/CMakeLists.txt

@@ -82,6 +82,7 @@ find_package(CURL REQUIRED)
 add_executable(${TARGET_NAME} main.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/file_logger.cc
+    ${CMAKE_CURRENT_SOURCE_DIR}/utils/modellist_utils.cc
   )
 
 target_link_libraries(${TARGET_NAME} PRIVATE httplib::httplib)

engine/commands/cortex_upd_cmd.h

@@ -127,7 +127,7 @@ inline bool ReplaceBinaryInflight(const std::filesystem::path& src,
                                           std::filesystem::perms::others_read |
                                           std::filesystem::perms::others_exec);
   } catch (const std::exception& e) {
-    CTL_ERR("Something wrong happened: " << e.what());
+    CTL_ERR("Something went wrong: " << e.what());
     if (std::filesystem::exists(temp)) {
       std::rename(temp.string().c_str(), dst.string().c_str());
       CLI_LOG("Restored binary file");

engine/commands/model_get_cmd.cc

@@ -4,21 +4,16 @@
 #include <vector>
 #include "cmd_info.h"
 #include "config/yaml_config.h"
-#include "trantor/utils/Logger.h"
-#include "utils/cortex_utils.h"
 #include "utils/file_manager_utils.h"
 #include "utils/logging_utils.h"
 
 namespace commands {
 
-ModelGetCmd::ModelGetCmd(std::string model_handle)
-    : model_handle_(std::move(model_handle)) {}
-
-void ModelGetCmd::Exec() {
+void ModelGetCmd::Exec(const std::string& model_handle) {
   auto models_path = file_manager_utils::GetModelsContainerPath();
   if (std::filesystem::exists(models_path) &&
       std::filesystem::is_directory(models_path)) {
-    CmdInfo ci(model_handle_);
+    CmdInfo ci(model_handle);
     std::string model_file =
         ci.branch == "main" ? ci.model_name : ci.model_name + "-" + ci.branch;
     bool found_model = false;
@@ -149,4 +144,4 @@ void ModelGetCmd::Exec() {
     CLI_LOG("Model not found!");
   }
 }
-};  // namespace commands
+};  // namespace commands

engine/commands/model_get_cmd.h

@@ -1,17 +1,10 @@
 #pragma once
 
-
-#include <cmath>  // For std::isnan
 #include <string>
 namespace commands {
 
 class ModelGetCmd {
  public:
-
-  ModelGetCmd(std::string model_handle);
-  void Exec();
-
- private:
-  std::string model_handle_;
+  void Exec(const std::string& model_handle);
 };
-}  // namespace commands
+}  // namespace commands

engine/commands/run_cmd.cc

@@ -2,13 +2,8 @@
 #include "chat_cmd.h"
 #include "cmd_info.h"
 #include "config/yaml_config.h"
-#include "engine_install_cmd.h"
-#include "httplib.h"
-#include "model_pull_cmd.h"
 #include "model_start_cmd.h"
 #include "server_start_cmd.h"
-#include "trantor/utils/Logger.h"
-#include "utils/cortex_utils.h"
 #include "utils/file_manager_utils.h"
 
 namespace commands {
@@ -46,7 +41,7 @@ void RunCmd::Exec() {
     if (!commands::IsServerAlive(host_, port_)) {
       CLI_LOG("Starting server ...");
       commands::ServerStartCmd ssc;
-      if(!ssc.Exec(host_, port_)) {
+      if (!ssc.Exec(host_, port_)) {
         return;
       }
     }

engine/config/gguf_parser.cc

@@ -409,7 +409,23 @@ void GGUFHandler::ModelConfigFromMetadata() {
   model_config_.created = std::time(nullptr);
   model_config_.model = "model";
   model_config_.owned_by = "";
-  model_config_.version;
+  model_config_.seed = -1;
+  model_config_.dynatemp_range = 0.0f;
+  model_config_.dynatemp_exponent = 1.0f;
+  model_config_.top_k = 40;
+  model_config_.min_p = 0.05f;
+  model_config_.tfs_z = 1.0f;
+  model_config_.typ_p = 1.0f;
+  model_config_.repeat_last_n = 64;
+  model_config_.repeat_penalty = 1.0f;
+  model_config_.mirostat = false;
+  model_config_.mirostat_tau = 5.0f;
+  model_config_.mirostat_eta = 0.1f;
+  model_config_.penalize_nl = false;
+  model_config_.ignore_eos = false;
+  model_config_.n_probs = 0;
+  model_config_.min_keep = 0;
+  model_config_.grammar = "";
 
   // Get version, bos, eos id, contex_len, ngl from meta data
   for (const auto& [key, value] : metadata_uint8_) {
@@ -522,7 +538,7 @@ void GGUFHandler::ModelConfigFromMetadata() {
   for (const auto& [key, value] : metadata_string_) {
     if (key.compare("general.name") == 0) {
       name = std::regex_replace(value, std::regex(" "), "-");
-    } else if (key.compare("tokenizer.chat_template") == 0) {
+    } else if (key.find("chat_template") != std::string::npos) {
       if (value.compare(ZEPHYR_JINJA) == 0) {
         chat_template =
             "<|system|>\n{system_message}</s>\n<|user|>\n{prompt}</"
@@ -564,12 +580,21 @@ void GGUFHandler::ModelConfigFromMetadata() {
     }
   }
 
-  eos_string = tokens[eos_token];
-  bos_string = tokens[bos_token];
-  stop.push_back(std::move(eos_string));
+  try {
+    if (tokens.size() > eos_token) {
+      eos_string = tokens[eos_token];
+      stop.push_back(std::move(eos_string));
+    } else {
+      LOG_ERROR << "Can't find stop token";
+    }
+  } catch (const std::exception& e) {
+    LOG_ERROR << "Can't find stop token";
+  }
 
   model_config_.stop = std::move(stop);
-
+  if (chat_template.empty())
+    chat_template =
+        "[INST] <<SYS>>\n{system_message}\n<</SYS>>\n{prompt}[/INST]";
   model_config_.prompt_template = std::move(chat_template);
   model_config_.name = name;
   model_config_.model = name;

engine/config/model_config.h

@@ -36,5 +36,23 @@ struct ModelConfig {
   std::size_t created;
   std::string object;
   std::string owned_by = "";
+
+  int seed = -1;
+  float dynatemp_range = 0.0f;
+  float dynatemp_exponent = 1.0f;
+  int top_k = 40;
+  float min_p = 0.05f;
+  float tfs_z = 1.0f;
+  float typ_p = 1.0f;
+  int repeat_last_n = 64;
+  float repeat_penalty = 1.0f;
+  bool mirostat = false;
+  float mirostat_tau = 5.0f;
+  float mirostat_eta = 0.1f;
+  bool penalize_nl = false;
+  bool ignore_eos = false;
+  int n_probs = 0;
+  int min_keep = 0;
+  std::string grammar;
 };
 }  // namespace config