Merge branch 'dev' of https://github.com/janhq/cortex into api-update

Author: irfanpena

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+## Describe Your Changes
+
+-
+
+## Fixes Issues
+
+- Closes #
+- Closes #
+
+## Self Checklist
+
+- [ ] Added relevant comments, esp in complex areas
+- [ ] Updated docs (for bug fixes / features)
+- [ ] Created issues for follow-up changes or refactoring needed

.github/workflows/cortex-cpp-build.yml

@@ -31,12 +31,11 @@ jobs:
           GITHUB_REF: ${{ github.ref }}
       - name: Create Draft Release
         id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: softprops/action-gh-release@v2
         with:
           tag_name: ${{ github.ref_name }}
-          release_name: "${{ env.VERSION }}"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: "${{ env.VERSION }}"
           draft: true
           prerelease: false
 
@@ -235,4 +234,28 @@ jobs:
           upload_url: ${{ needs.create-draft-release.outputs.upload_url }}
           asset_path: ./cortex-cpp/cortex-cpp.tar.gz
           asset_name: cortex-cpp-${{ needs.create-draft-release.outputs.version }}-${{ matrix.os }}-${{ matrix.name }}.tar.gz
-          asset_content_type: application/gzip
+          asset_content_type: application/gzip
+
+  update_release_draft:
+    needs: [build-and-test]
+    permissions:
+      # write permission is required to create a github release
+      contents: write
+      # write permission is required for autolabeler
+      # otherwise, read permission is required at least
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      # (Optional) GitHub Enterprise requires GHE_HOST variable set
+      #- name: Set GHE_HOST
+      #  run: |
+      #    echo "GHE_HOST=${GITHUB_SERVER_URL##https:\/\/}" >> $GITHUB_ENV
+
+      # Drafts your next Release notes as Pull Requests are merged into "master"
+      - uses: release-drafter/release-drafter@v5
+        # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml
+        # with:
+        #   config-name: my-config.yml
+        #   disable-autolabeler: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -1,133 +1,157 @@
 # Cortex Monorepo
 
-This monorepo contains two projects: CortexJS and CortexCPP.
-
-## CortexJS: Stateful Business Backend
-
-* All of the stateful endpoints:
-	+ /threads
-	+ /messages
-	+ /models
-	+ /runs
-	+ /vector_store
-	+ /settings
-	+ /?auth
-	+ …
-* Database & Filesystem
-* API Gateway
-* Authentication & Authorization
-* Observability
-
-## CortexCPP: Stateless Embedding Backend
-
-* All of the high performance, stateless endpoints:
-	+ /chat/completion
-	+ /audio
-	+ /fine_tuning
-	+ /embeddings
-	+ /load_model
-	+ /unload_model
-* Kernel - Hardware Recognition
-
-## Project Structure
-
-```
-.
-├── cortex-js/
-│   ├── package.json
-│   ├── README.md
-│   ├── Dockerfile
-│   ├── docker-compose.yml
-│   ├── src/
-│   │   ├── controllers/
-│   │   ├── modules/
-│   │   ├── services/
-│   │   └── ...
-│   └── ...
-├── cortex-cpp/
-│   ├── app/
-│   │   ├── controllers/
-│   │   ├── models/
-│   │   ├── services/
-│   │   ├── ?engines/
-│   │   │   ├── llama.cpp
-│   │   │   ├── tensorrt-llm
-│   │   │   └── ...
-│   │   └── ...
-│   ├── CMakeLists.txt
-│   ├── config.json
-│   ├── Dockerfile
-│   ├── docker-compose.yml
-│   ├── README.md
-│   └── ...
-├── scripts/
-│   └── ...
-├── README.md
-├── package.json
-├── Dockerfile
-├── docker-compose.yml
-└── docs/
-    └── ...
-```
-
-## Installation
-
-### NPM Install
-
-* Pre-install script:
+# Installation
+
+## Prerequisites
+
+### **Dependencies**
+
+Before installation, ensure that you have installed the following:
+
+- **Node.js**: Required for running the installation.
+- **NPM**: Needed to manage packages.
+- **CPU Instruction Sets**: Available for download from the [Cortex GitHub Releases](https://github.com/janhq/cortex/releases) page.
+
+<aside>
+💡 The **CPU instruction sets** are not required for the initial installation of Cortex. This dependency will be automatically installed during the Cortex initialization if they are not already on your system.
+
+</aside>
+
+### **Hardware**
+
+Ensure that your system meets the following requirements to run Cortex:
+
+- **OS**:
+  - MacOSX 13.6 or higher.
+  - Windows 10 or higher.
+  - Ubuntu 12.04 and later.
+- **RAM (CPU Mode):**
+  - 8GB for running up to 3B models.
+  - 16GB for running up to 7B models.
+  - 32GB for running up to 13B models.
+- **VRAM (GPU Mode):**
+
+  - 6GB can load the 3B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU.
+  - 8GB can load the 7B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU.
+  - 12GB can load the 13B model (int4) with `ngl` at 120 ~ full speed on CPU/ GPU.
+
+- **Disk**: At least 10GB for app and model download.
+
+## Cortex Installation
+
+To install Cortex, follow the steps below:
+
+### Step 1: Install Cortex
+
+Run the following command to install Cortex globally on your machine:
+
+```bash
+# Install using NPM globally
+npm i -g @janhq/cortex
+```
+
+### Step 2: Verify the Installation
+
+After installation, you can verify that Cortex is installed correctly by getting help information.
+
+```bash
+# Get the help information
+cortex -h
+```
+
+### Step 3: Initialize Cortex
+
+Once verified, you need to initialize the Cortex engine.
+
+1. Initialize the Cortex engine:
+
+```
+cortex init
+```
+
+1. Select between `CPU` and `GPU` modes.
+
 ```bash
-npm pre-install script; platform specific (MacOS / Windows / Linux)
+? Select run mode (Use arrow keys)
+> CPU
+  GPU
 ```
-* Tag based:
-```json
-npm install @janhq/cortex
-npm install @janhq/cortex#cuda
-npm install @janhq/cortex#cuda-avx512
-npm install @janhq/cortex#cuda-avx
+
+2. Select between GPU types.
+
+```bash
+? Select GPU types (Use arrow keys)
+> Nvidia
+  Others (Vulkan)
 ```
 
-### CLI Install Script
+3. Select CPU instructions (will be deprecated soon).
 
 ```bash
-cortex init (AVX2 + Cuda)
+? Select CPU instructions (Use arrow keys)
+> AVX2
+  AVX
+  AVX-512
+```
 
-Enable GPU Acceleration?
-1. Nvidia (default) - detected
-2. AMD
-3. Mac Metal
+1. Cortex will download the required CPU instruction sets if you choose `CPU` mode. If you choose `GPU` mode, Cortex will download the necessary dependencies to use your GPU.
+2. Once downloaded, Cortex is ready to use!
 
-Enter your choice: 
+### Step 4: Pull a model
 
-CPU Instructions
-1. AVX2 (default) - Recommend based on what the user has
-2. AVX  (old CPU)
-3. AVX512
+From HuggingFace
 
-Enter your choice: 
+```bash
+cortex pull janhq/phi-3-medium-128k-instruct-GGUF
+```
 
-Downloading cortex-cuda-avx.so........................25%
+From Jan Hub (TBD)
 
-Cortex is ready!
+```bash
+cortex pull llama3
+```
 
-It seems like you have installed models from other applications. Do you want to import them?
-1. Import from /Users/HOME/jan/models
-2. Import from /Users/HOME/lmstudio/models
-3. Import everything
+### Step 5: Chat
 
-Importing from /Users/HOME/jan/models..................17%
+```bash
+cortex run janhq/phi-3-medium-128k-instruct-GGUF
 ```
 
-## Backend (jan app)
+## Run as an API server
 
-```json
-POST /settings
-{
-    "gpu_enabled": true,
-    "gpu_family": "Nvidia",
-    "cpu_instructions": "AVX2" 
-}
+```bash
+cortex serve
 ```
 
-## Client Library Configuration
+## Build from Source
 
-TBD
+To install Cortex from the source, follow the steps below:
+
+1. Clone the Cortex repository [here](https://github.com/janhq/cortex/tree/dev).
+2. Navigate to the `cortex-js` folder.
+3. Open the terminal and run the following command to build the Cortex project:
+
+```bash
+npx nest build
+```
+
+1. Make the `command.js` executable:
+
+```bash
+chmod +x '[path-to]/cortex/cortex-js/dist/src/command.js'
+```
+
+1. Link the package globally:
+
+```bash
+npm link
+```
+
+## Uninstall Cortex
+
+Run the following command to uninstall Cortex globally on your machine:
+
+```
+# Uninstall globally using NPM
+npm uninstall -g @janhq/cortex
+```

cortex-cpp/common/base.h

@@ -16,6 +16,9 @@ class BaseModel {
   virtual void ModelStatus(
       const HttpRequestPtr& req,
       std::function<void(const HttpResponsePtr&)>&& callback) = 0;
+  
+  virtual void GetModels(const HttpRequestPtr& req,
+      std::function<void(const HttpResponsePtr&)>&& callback) = 0;
 };
 
 class BaseChatCompletion {

cortex-cpp/controllers/server.cc

@@ -129,6 +129,40 @@ void server::ModelStatus(
   LOG_TRACE << "Done get model status";
 }
 
+void server::GetModels(const HttpRequestPtr& req,
+                       std::function<void(const HttpResponsePtr&)>&& callback) {
+  if (!IsEngineLoaded()) {
+    Json::Value res;
+    res["message"] = "Engine is not loaded yet";
+    auto resp = cortex_utils::nitroHttpJsonResponse(res);
+    resp->setStatusCode(k409Conflict);
+    callback(resp);
+    LOG_WARN << "Engine is not loaded yet";
+    return;
+  }
+
+  LOG_TRACE << "Start to get models";
+  if (engine_->IsSupported("GetModels")) {
+    engine_->GetModels(
+        req->getJsonObject(),
+        [cb = std::move(callback)](Json::Value status, Json::Value res) {
+          auto resp = cortex_utils::nitroHttpJsonResponse(res);
+          resp->setStatusCode(static_cast<drogon::HttpStatusCode>(
+              status["status_code"].asInt()));
+          cb(resp);
+        });
+  } else {
+    Json::Value res;
+    res["message"] = "Method is not supported yet";
+    auto resp = cortex_utils::nitroHttpJsonResponse(res);
+    resp->setStatusCode(k500InternalServerError);
+    callback(resp);
+    LOG_WARN << "Method is not supported yet";
+  }
+
+  LOG_TRACE << "Done get models";
+}
+
 void server::LoadModel(const HttpRequestPtr& req,
                        std::function<void(const HttpResponsePtr&)>&& callback) {
   auto engine_type =
@@ -144,10 +178,9 @@ void server::LoadModel(const HttpRequestPtr& req,
     };
 
     try {
-      std::string abs_path = cortex_utils::GetCurrentPath() +
-                             get_engine_path(cur_engine_name_);
-      dylib_ =
-          std::make_unique<cortex_cpp::dylib>(abs_path, "engine");
+      std::string abs_path =
+          cortex_utils::GetCurrentPath() + get_engine_path(cur_engine_name_);
+      dylib_ = std::make_unique<cortex_cpp::dylib>(abs_path, "engine");
     } catch (const cortex_cpp::dylib::load_error& e) {
       LOG_ERROR << "Could not load engine: " << e.what();
       dylib_.reset();