Merge pull request #1700 from janhq/j/proxy-docs

namchuai · web-flow · commit 53cfc42f45bd · 2024-11-19T11:20:20.000+07:00
docs: add proxy docs
diff --git a/docs/docs/configurations/index.mdx b/docs/docs/configurations/index.mdx
@@ -8,5 +8,6 @@ title: Cortex configurations
 Welcome to the Cortex configurations documentation. Here you will find detailed guides and references for configuring various aspects of Cortex, including:
 
 - **CORS**: Learn how to set up Cross-Origin Resource Sharing.
+- **Proxy**: Configure the proxy for Cortex.
 
 Use the sidebar to navigate through the different configuration topics.
diff --git a/docs/docs/configurations/proxy.mdx b/docs/docs/configurations/proxy.mdx
@@ -0,0 +1,207 @@
+---
+title: Proxy
+description: Setting up Proxy
+slug: "proxy"
+---
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
+:::warning
+🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase.
+:::
+
+# Proxy Configuration Guide
+
+This document describes how to configure proxy settings for Cortex to be able to work behind your proxy.
+
+## Command Line Interface (CLI)
+
+### Basic Usage
+
+```bash
+cortex config [OPTIONS] [COMMAND]
+```
+
+### Commands
+
+- `status`: Display all current configurations
+
+```bash
+cortex config status
+```
+
+Example Output:
+
+```bash
++-----------------------+------------------------+
+| Config name           | Value                  |
++-----------------------+------------------------+
+| no_proxy              | localhost,127.0.0.1    |
++-----------------------+------------------------+
+| proxy_password        |                        |
++-----------------------+------------------------+
+| proxy_url             | http://localhost:8080  |
++-----------------------+------------------------+
+| proxy_username        |                        |
++-----------------------+------------------------+
+| verify_host_ssl       | true                   |
++-----------------------+------------------------+
+| verify_peer_ssl       | false                  |
++-----------------------+------------------------+
+| verify_proxy_host_ssl | true                   |
++-----------------------+------------------------+
+| verify_proxy_ssl      | true                   |
++-----------------------+------------------------+
+```
+
+### Options
+
+| Option                              | Description                 | Example                                           |
+| ----------------------------------- | --------------------------- | ------------------------------------------------- |
+| `-h, --help`                        | Print help message and exit |
+| `--proxy_url <proxy_url>`           | Set the proxy URL           | `cortex config --proxy_url http://localhost:8080` |
+| `--proxy_username <proxy_username>` | Set the username for proxy  | `cortex config --proxy_username my_username`      |
+| `--proxy_password <proxy_password>` | Set the password for proxy  | `cortex config --proxy_password my_password`      |
+| `--no_proxy <no_proxy>`             | Set the no_proxy list       | `cortex config --no_proxy localhost,127.0.0.1`    |
+| `--verify_proxy_ssl [on/off]`       | Verify proxy SSL            | `cortex config --verify_proxy_ssl on`             |
+| `--verify_proxy_host_ssl [on/off]`  | Verify proxy host SSL       | `cortex config --verify_proxy_host_ssl on`        |
+| `--verify_peer_ssl [on/off]`        | Verify peer SSL             | `cortex config --verify_peer_ssl off`             |
+| `--verify_host_ssl [on/off]`        | Verify host SSL             | `cortex config --verify_host_ssl on`              |
+
+## Proxy API Configuration
+
+### Endpoints
+
+#### Get Current Configuration
+
+```http
+GET /v1/configs
+```
+
+Retrieves the current configuration settings.
+
+##### Response
+
+```json
+{
+  "allowed_origins": [
+    "http://localhost:39281",
+    "http://127.0.0.1:39281",
+    "http://0.0.0.0:39281"
+  ],
+  "cors": true,
+  "no_proxy": "localhost,127.0.0.1",
+  "proxy_password": "",
+  "proxy_url": "http://localhost:8080",
+  "proxy_username": "",
+  "verify_host_ssl": true,
+  "verify_peer_ssl": false,
+  "verify_proxy_host_ssl": true,
+  "verify_proxy_ssl": true
+}
+```
+
+#### Update Configuration
+
+```http
+PATCH /v1/configs
+```
+
+Updates proxy configuration settings.
+
+##### Request Headers
+
+```
+Content-Type: application/json
+```
+
+##### Request Body
+
+```json
+{
+  "no_proxy": "localhost",
+  "proxy_url": "http://localhost:8080",
+  "proxy_username": "my_username",
+  "proxy_password": "my_password",
+  "verify_host_ssl": false,
+  "verify_peer_ssl": false,
+  "verify_proxy_host_ssl": false,
+  "verify_proxy_ssl": false
+}
+```
+
+##### Parameters
+
+| Field                   | Type    | Description                                                                            |
+| ----------------------- | ------- | -------------------------------------------------------------------------------------- |
+| `no_proxy`              | string  | List of origins which request do not need to go through a proxy. Comma separated value |
+| `proxy_url`             | string  | Proxy URL                                                                              |
+| `proxy_username`        | string  | Username for proxy authentication                                                      |
+| `proxy_password`        | string  | Password for proxy authentication                                                      |
+| `verify_host_ssl`       | boolean | Verify host SSL                                                                        |
+| `verify_peer_ssl`       | boolean | Verify peer SSL                                                                        |
+| `verify_proxy_host_ssl` | boolean | Verify proxy host SSL                                                                  |
+| `verify_proxy_ssl`      | boolean | Verify proxy SSL                                                                       |
+
+##### Response
+
+```json
+{
+  "config": {
+    "allowed_origins": [
+      "http://localhost:39281",
+      "http://127.0.0.1:39281",
+      "http://0.0.0.0:39281"
+    ],
+    "cors": true,
+    "no_proxy": "localhost",
+    "proxy_password": "my_password",
+    "proxy_url": "http://localhost:8080",
+    "proxy_username": "my_username",
+    "verify_host_ssl": false,
+    "verify_peer_ssl": false,
+    "verify_proxy_host_ssl": false,
+    "verify_proxy_ssl": false
+  },
+  "message": "Configuration updated successfully"
+}
+```
+
+## Testing proxy configuration
+
+You can test your proxy configuration using [mitmproxy](https://docs.mitmproxy.org/stable). This guide is written on macOS, but you can use it on any other platform.
+
+### Install mitmproxy
+
+```bash
+brew install mitmproxy
+```
+
+### Start mitmproxy
+
+```bash
+mitmproxy --set stream_large_bodies=1m
+```
+
+mitmproxy will start on port `8080`. After mitmproxy started, you can adding options by pressing `O`. mitmproxy will display an option screen. You can check their document to learn more about mitmproxy. But let's take a simple option for now by setting the `proxyauth` for our local proxy. Inside the option screen, search for `proxyauth` and hit enter. Then, type `username:password` and hit enter again. You will see your newly added option is red-colored.
+
+### Configuring Cortex to use that proxy
+
+Let's using CLI to configure Cortex to use that proxy.
+
+```bash
+cortex config --proxy_url http://localhost:8080 --proxy_username username --proxy_password password
+```
+
+### Testing the proxy
+
+Now, let's test the proxy. If you are setting the username and password correctly (same with `proxyauth` in mitmproxy), you will see the request in mitmproxy. For example, command `cortex pull tinyllama` should be successfully and returns a list of selectable models. Also, you will see your request in mitmproxy CLI screen.
+
+Let's try to use a wrong authentication for your proxy.
+
+```bash
+cortex config --proxy_password wrong_pw
+```
+
+Now, let's test the proxy again. You will see the request is failed and returns an error.
diff --git a/docs/sidebars.ts b/docs/sidebars.ts
@@ -95,6 +95,11 @@ const sidebars: SidebarsConfig = {
           id: "configurations/cors",
           label: "CORS",
         },
+        {
+          type: "doc",
+          id: "configurations/proxy",
+          label: "Proxy",
+        },
       ],
     },
     {
diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc
@@ -10,6 +10,7 @@
 #include "utils/format_utils.h"
 #include "utils/logging_utils.h"
 #include "utils/result.hpp"
+#include "utils/string_utils.h"
 
 namespace {
 size_t WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) {
@@ -75,6 +76,10 @@ void SetUpProxy(CURL* handle, std::shared_ptr<ConfigService> config_service) {
       CTL_INF("Verify host ssl: " << verify_host_ssl);
 
       curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str());
+      if (string_utils::StartsWith(proxy_url, "https")) {
+        curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_HTTPS);
+      }
+
       curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L);
       curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST,
                        verify_host_ssl ? 2L : 0L);
diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h
@@ -57,7 +57,7 @@ constexpr const auto kDefaultLatestLlamacppRelease = "";
 constexpr const auto kDefaultCorsEnabled = true;
 const std::vector<std::string> kDefaultEnabledOrigins{
     "http://localhost:39281", "http://127.0.0.1:39281", "http://0.0.0.0:39281"};
-constexpr const auto kDefaultNoProxy = "localhost,127.0.0.1";
+constexpr const auto kDefaultNoProxy = "example.com,::1,localhost,127.0.0.1";
 
 inline cpp::result<void, std::string> DumpYamlConfig(const CortexConfig& config,
                                                      const std::string& path) {
diff --git a/engine/utils/curl_utils.h b/engine/utils/curl_utils.h
@@ -10,6 +10,7 @@
 #include "utils/file_manager_utils.h"
 #include "utils/logging_utils.h"
 #include "utils/result.hpp"
+#include "utils/string_utils.h"
 #include "utils/url_parser.h"
 
 enum class RequestType { GET, PATCH, POST, DEL };
@@ -24,7 +25,7 @@ size_t WriteCallback(void* contents, size_t size, size_t nmemb,
   return totalSize;
 }
 
-void SetUpProxy(CURL* handle) {
+void SetUpProxy(CURL* handle, const std::string& url) {
   auto config = file_manager_utils::GetCortexConfig();
   if (!config.proxyUrl.empty()) {
     auto proxy_url = config.proxyUrl;
@@ -39,13 +40,18 @@ void SetUpProxy(CURL* handle) {
     auto no_proxy = config.noProxy;
 
     CTL_INF("=== Proxy configuration ===");
+    CTL_INF("Request url: " << url);
     CTL_INF("Proxy url: " << proxy_url);
     CTL_INF("Verify proxy ssl: " << verify_proxy_ssl);
     CTL_INF("Verify proxy host ssl: " << verify_proxy_host_ssl);
     CTL_INF("Verify ssl: " << verify_ssl);
     CTL_INF("Verify host ssl: " << verify_host_ssl);
+    CTL_INF("No proxy: " << no_proxy);
 
     curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str());
+    if (string_utils::StartsWith(proxy_url, "https")) {
+      curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_HTTPS);
+    }
     curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L);
     curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, verify_host_ssl ? 2L : 0L);
 
@@ -88,7 +94,7 @@ inline cpp::result<std::string, std::string> SimpleGet(const std::string& url,
 
   std::string readBuffer;
 
-  SetUpProxy(curl);
+  SetUpProxy(curl, url);
   curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
   curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
   curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
@@ -139,7 +145,7 @@ inline cpp::result<std::string, std::string> SimpleRequest(
   }
   std::string readBuffer;
 
-  SetUpProxy(curl);
+  SetUpProxy(curl, url);
   curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers);
   curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
   if (request_type == RequestType::PATCH) {
