continuedev · sestinj · May 30, 2024 · May 3, 2024 · May 3, 2024 · May 4, 2024
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -97,7 +97,7 @@ Pre-requisite: You should use the Intellij IDE, which can be downloaded [here](h
 2. Run `scripts/install-dependencies.sh` or `scripts/install-dependencies.ps1` on Windows. This will install and build all of the necessary dependencies.
 3. To test the plugin, select the "Run Plugin" Gradle configuration and click the "Run" or "Debug" button as shown in this screenshot:
    ![img](./media/IntelliJRunPluginScreenshot.png)
-4. To package the extension, choose the "Build Plugin" Gradle configuration. This will generate a .zip file in `extensions/intellij/build/distributions` with the version defined in `extensions/intellij/gradle.properties`.
+4. To package the extension, run `./gradlew build` (or `./gradlew.bat build` on Windows) from the `extensions/intellij` directory. This will generate a .zip file in `extensions/intellij/build/distributions` with the version defined in `extensions/intellij/gradle.properties`.
 5. If you make changes, you may need to re-build before running the "Build Plugin" configuration
 
    a. If you change code from the `core` or `binary` directories, make sure to run `npm run build` from the `binary` directory to create a new binary.

diff --git a/core/autocomplete/templates.ts b/core/autocomplete/templates.ts
@@ -24,6 +24,13 @@ const stableCodeFimTemplate: AutocompleteTemplate = {
   },
 };
 
+const codestralFimTemplate: AutocompleteTemplate = {
+  template: "<s>[SUFFIX]{{{suffix}}}[PREFIX]{{{prefix}}}",
+  completionOptions: {
+    stop: ["[PREFIX]", "[SUFFIX]"],
+  },
+};
+
 const codegemmaFimTemplate: AutocompleteTemplate = {
   template:
     "<|fim_prefix|>{{{prefix}}}<|fim_suffix|>{{{suffix}}}<|fim_middle|>",
@@ -227,6 +234,10 @@ export function getTemplateForModel(model: string): AutocompleteTemplate {
     return stableCodeFimTemplate;
   }
 
+  if (lowerCaseModel.includes("codestral")) {
+    return codestralFimTemplate;
+  }
+
   if (lowerCaseModel.includes("codegemma")) {
     return codegemmaFimTemplate;
   }

diff --git a/core/index.d.ts b/core/index.d.ts
@@ -304,6 +304,9 @@ export interface LLMOptions {
 
   useLegacyCompletionsEndpoint?: boolean;
 
+  // Cloudflare options
+  accountId?: string;
+
   // Azure options
   engine?: string;
   apiVersion?: string;
@@ -539,7 +542,8 @@ type ModelProvider =
   | "groq"
   | "continue-proxy"
   | "fireworks"
-  | "custom";
+  | "custom"
+  | "cloudflare";
 
 export type ModelName =
   | "AUTODETECT"

diff --git a/core/llm/index.ts b/core/llm/index.ts
@@ -99,6 +99,7 @@ export abstract class BaseLLM implements ILLM {
   apiType?: string;
   region?: string;
   projectId?: string;
+  accountId?: string;
 
   private _llmOptions: LLMOptions;
 
@@ -150,6 +151,7 @@ export abstract class BaseLLM implements ILLM {
     if (this.apiBase && !this.apiBase.endsWith("/")) {
       this.apiBase = `${this.apiBase}/`;
     }
+    this.accountId = options.accountId;
 
     this.engine = options.engine;
     this.apiVersion = options.apiVersion;

diff --git a/core/llm/llms/Cloudflare.ts b/core/llm/llms/Cloudflare.ts
@@ -0,0 +1,62 @@
+import { BaseLLM } from "../index.js";
+import { ChatMessage, CompletionOptions, ModelProvider } from "../../index.js";
+import { stripImages } from "../countTokens.js";
+import { streamSse } from "../stream.js";
+
+export default class Cloudflare extends BaseLLM {
+  static providerName: ModelProvider = "cloudflare";
+
+  private _convertArgs(options: CompletionOptions) {
+    const finalOptions = {
+      max_tokens: options.maxTokens,
+    };
+
+    return finalOptions;
+  }
+
+  protected async *_streamChat(
+    messages: ChatMessage[],
+    options: CompletionOptions,
+  ): AsyncGenerator<ChatMessage, any, unknown> {
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
+      ...this.requestOptions?.headers,
+    };
+
+    const resp = await this.fetch(
+      new URL(
+        `https://api.cloudflare.com/client/v4/accounts/${this.accountId}/ai/v1/chat/completions`,
+      ),
+      {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+          messages,
+          stream: true,
+          model: this.model,
+          ...this._convertArgs(options),
+        }),
+      },
+    );
+
+    for await (const value of streamSse(resp)) {
+      console.log(value);
+      if (value.choices?.[0]?.delta?.content) {
+        yield value.choices[0].delta;
+      }
+    }
+  }
+
+  protected async *_streamComplete(
+    prompt: string,
+    options: CompletionOptions,
+  ): AsyncGenerator<string> {
+    for await (const chunk of this._streamChat(
+      [{ role: "user", content: prompt }],
+      options,
+    )) {
+      yield stripImages(chunk.content);
+    }
+  }
+}
diff --git a/core/llm/llms/index.ts b/core/llm/llms/index.ts
@@ -29,6 +29,7 @@ import Replicate from "./Replicate.js";
 import TextGenWebUI from "./TextGenWebUI.js";
 import Together from "./Together.js";
 import ContinueProxy from "./stubs/ContinueProxy.js";
+import Cloudflare from "./Cloudflare";
 
 function convertToLetter(num: number): string {
   let result = "";
@@ -99,6 +100,7 @@ const LLMs = [
   Groq,
   Fireworks,
   ContinueProxy,
+  Cloudflare,
 ];
 
 export async function llmFromDescription(

diff --git a/docs/docs/reference/Model Providers/cloudflare.md b/docs/docs/reference/Model Providers/cloudflare.md
@@ -0,0 +1,46 @@
+# Cloudflare Workers AI
+
+Cloudflare Workers AI can be used for both chat and tab autocompletion in Continue. To setup Cloudflare Workers AI, add the following to your `config.json` file:
+
+```json title="~/.continue/config.json"
+{
+  "models": [
+    {
+      "accountId": "YOUR CLOUDFLARE ACCOUNT ID",
+      "apiKey": "YOUR CLOUDFLARE API KEY",
+      "contextLength": 2400,
+      "completionOptions": {
+        "maxTokens": 500
+      },
+      "model": "@cf/meta/llama-3-8b-instruct", // This can be the name of any model supported by Workers AI
+      "provider": "cloudflare",
+      "title": "Llama 3 8B"
+    },
+    {
+      "accountId": "YOUR CLOUDFLARE ACCOUNT ID",
+      "apiKey": "YOUR CLOUDFLARE API KEY",
+      "contextLength": 2400,
+      "completionOptions": {
+        "maxTokens": 500
+      },
+      "model": "@hf/thebloke/deepseek-coder-6.7b-instruct-awq",
+      "provider": "cloudflare",
+      "title": "DeepSeek Coder 6.7b Instruct"
+    }
+    ...
+    "tabAutocompleteModel": {
+      "accountId": "YOUR CLOUDFLARE ACCOUNT ID",
+      "apiKey": "YOUR CLOUDFLARE API KEY",
+      "model": "@hf/thebloke/deepseek-coder-6.7b-base-awq",
+      "provider": "cloudflare",
+      "title": "DeepSeek 7b"
+    },
+  ]
+}
+```
+
+Visit the [Cloudflare dashboard](https://dash.cloudflare.com/) to [create an API key](https://developers.cloudflare.com/fundamentals/api/get-started/create-token/).
+
+Review [available models](https://developers.cloudflare.com/workers-ai/models/) on Workers AI
+
+[View the source](https://github.com/continuedev/continue/blob/main/core/llm/llms/Cloudflare.ts)
diff --git a/docs/docs/walkthroughs/set-up-codestral.md b/docs/docs/walkthroughs/set-up-codestral.md
@@ -14,7 +14,7 @@ keywords: [codestral, mistral, model setup]
 
 2. Click on the gear icon in the bottom right corner of the Continue window to open `~/.continue/config.json` (MacOS) / `%userprofile%\.continue\config.json` (Windows)
 
-3. Log in and create an API key on Mistral AI's La Plateforme [here](https://console.mistral.ai/codestral)
+3. Log in and create an API key on Mistral AI's La Plateforme [here](https://console.mistral.ai/codestral). Make sure you get an API key from the "Codestral" page, as an API key for the normal "api.mistral.ai" API will not work.
 
 4. To use Codestral as your model for both `autocomplete` and `chat`, replace `[API_KEY]` with your Mistral API key below and add it to your `config.json` file:
 

diff --git a/docs/static/schemas/config.json b/docs/static/schemas/config.json
@@ -150,7 +150,8 @@
             "flowise",
             "groq",
             "fireworks",
-            "continue-proxy"
+            "continue-proxy",
+            "cloudflare"
           ],
           "markdownEnumDescriptions": [
             "### OpenAI\nUse gpt-4, gpt-3.5-turbo, or any other OpenAI model. See [here](https://openai.com/product#made-for-developers) to obtain an API key.\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/openai)",
@@ -169,7 +170,8 @@
             "### Llamafile\nTo get started with llamafiles, find and download a binary on their [GitHub repo](https://github.com/Mozilla-Ocho/llamafile#binary-instructions). Then run it with the following command:\n\n```shell\nchmod +x ./llamafile\n./llamafile\n```\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/llamafile)",
             "### Mistral API\n\nTo get access to the Mistral API, obtain your API key from the [Mistral platform](https://docs.mistral.ai/)",
             "### DeepInfra\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/deepinfra)",
-            "### Continue Proxy\n\nContinue Enterprise users can use the Continue Proxy as a single point of access to models."
+            "### Continue Proxy\n\nContinue Enterprise users can use the Continue Proxy as a single point of access to models.",
+            "### Cloudflare Workers AI\n\n[Reference](https://developers.cloudflare.com/workers-ai/)"
           ],
           "type": "string"
         },
@@ -313,7 +315,8 @@
                   "gemini",
                   "huggingface-inference-api",
                   "replicate",
-                  "together"
+                  "together",
+                  "cloudflare"
                 ]
               }
             },
@@ -359,6 +362,64 @@
             }
           }
         },
+        {
+          "if": {
+            "properties": {
+              "provider": {
+                "enum": ["cloudflare"]
+              }
+            },
+            "required": ["provider"]
+          },
+          "then": {
+            "properties": {
+              "accountId": {
+                "type": "string"
+              },
+              "model": {
+                "anyOf": [
+                  {
+                    "enum": [
+                      "@cf/meta/llama-3-8b-instruct",
+                      "@hf/thebloke/deepseek-coder-6.7b-instruct-awq",
+                      "@cf/deepseek-ai/deepseek-math-7b-instruct",
+                      "@cf/thebloke/discolm-german-7b-v1-awq",
+                      "@cf/tiiuae/falcon-7b-instruct",
+                      "@cf/google/gemma-2b-it-lora",
+                      "@hf/google/gemma-7b-it",
+                      "@cf/google/gemma-7b-it-lora",
+                      "@hf/nousresearch/hermes-2-pro-mistral-7b",
+                      "@cf/meta/llama-2-7b-chat-fp16",
+                      "@cf/meta/llama-2-7b-chat-int8",
+                      "@cf/meta-llama/llama-2-7b-chat-hf-lora",
+                      "@hf/thebloke/llama-2-13b-chat-awq",
+                      "@hf/thebloke/llamaguard-7b-awq",
+                      "@cf/mistral/mistral-7b-instruct-v0.1",
+                      "@hf/mistral/mistral-7b-instruct-v0.2",
+                      "@cf/mistral/mistral-7b-instruct-v0.2-lora",
+                      "@hf/thebloke/neural-chat-7b-v3-1-awq",
+                      "@cf/openchat/openchat-3.5-0106",
+                      "@hf/thebloke/openhermes-2.5-mistral-7b-awq",
+                      "@cf/microsoft/phi-2",
+                      "@cf/qwen/qwen1.5-0.5b-chat",
+                      "@cf/qwen/qwen1.5-1.8b-chat",
+                      "@cf/qwen/qwen1.5-7b-chat-awq",
+                      "@cf/qwen/qwen1.5-14b-chat-awq",
+                      "@cf/defog/sqlcoder-7b-2",
+                      "@hf/nexusflow/starling-lm-7b-beta",
+                      "@cf/tinyllama/tinyllama-1.1b-chat-v1.0",
+                      "@hf/thebloke/zephyr-7b-beta-awq",
+                      "@hf/thebloke/deepseek-coder-6.7b-base-awq"
+                    ]
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              }
+            }
+          }
+        },
         {
           "if": {
             "properties": {

diff --git a/extensions/intellij/src/main/resources/config_schema.json b/extensions/intellij/src/main/resources/config_schema.json
@@ -150,7 +150,8 @@
             "flowise",
             "groq",
             "fireworks",
-            "continue-proxy"
+            "continue-proxy",
+            "cloudflare"
           ],
           "markdownEnumDescriptions": [
             "### OpenAI\nUse gpt-4, gpt-3.5-turbo, or any other OpenAI model. See [here](https://openai.com/product#made-for-developers) to obtain an API key.\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/openai)",
@@ -169,7 +170,8 @@
             "### Llamafile\nTo get started with llamafiles, find and download a binary on their [GitHub repo](https://github.com/Mozilla-Ocho/llamafile#binary-instructions). Then run it with the following command:\n\n```shell\nchmod +x ./llamafile\n./llamafile\n```\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/llamafile)",
             "### Mistral API\n\nTo get access to the Mistral API, obtain your API key from the [Mistral platform](https://docs.mistral.ai/)",
             "### DeepInfra\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/deepinfra)",
-            "### Continue Proxy\n\nContinue Enterprise users can use the Continue Proxy as a single point of access to models."
+            "### Continue Proxy\n\nContinue Enterprise users can use the Continue Proxy as a single point of access to models.",
+            "### Cloudflare Workers AI\n\n[Reference](https://developers.cloudflare.com/workers-ai/)"
           ],
           "type": "string"
         },
@@ -313,7 +315,8 @@
                   "gemini",
                   "huggingface-inference-api",
                   "replicate",
-                  "together"
+                  "together",
+                  "cloudflare"
                 ]
               }
             },
@@ -359,6 +362,64 @@
             }
           }
         },
+        {
+          "if": {
+            "properties": {
+              "provider": {
+                "enum": ["cloudflare"]
+              }
+            },
+            "required": ["provider"]
+          },
+          "then": {
+            "properties": {
+              "accountId": {
+                "type": "string"
+              },
+              "model": {
+                "anyOf": [
+                  {
+                    "enum": [
+                      "@cf/meta/llama-3-8b-instruct",
+                      "@hf/thebloke/deepseek-coder-6.7b-instruct-awq",
+                      "@cf/deepseek-ai/deepseek-math-7b-instruct",
+                      "@cf/thebloke/discolm-german-7b-v1-awq",
+                      "@cf/tiiuae/falcon-7b-instruct",
+                      "@cf/google/gemma-2b-it-lora",
+                      "@hf/google/gemma-7b-it",
+                      "@cf/google/gemma-7b-it-lora",
+                      "@hf/nousresearch/hermes-2-pro-mistral-7b",
+                      "@cf/meta/llama-2-7b-chat-fp16",
+                      "@cf/meta/llama-2-7b-chat-int8",
+                      "@cf/meta-llama/llama-2-7b-chat-hf-lora",
+                      "@hf/thebloke/llama-2-13b-chat-awq",
+                      "@hf/thebloke/llamaguard-7b-awq",
+                      "@cf/mistral/mistral-7b-instruct-v0.1",
+                      "@hf/mistral/mistral-7b-instruct-v0.2",
+                      "@cf/mistral/mistral-7b-instruct-v0.2-lora",
+                      "@hf/thebloke/neural-chat-7b-v3-1-awq",
+                      "@cf/openchat/openchat-3.5-0106",
+                      "@hf/thebloke/openhermes-2.5-mistral-7b-awq",
+                      "@cf/microsoft/phi-2",
+                      "@cf/qwen/qwen1.5-0.5b-chat",
+                      "@cf/qwen/qwen1.5-1.8b-chat",
+                      "@cf/qwen/qwen1.5-7b-chat-awq",
+                      "@cf/qwen/qwen1.5-14b-chat-awq",
+                      "@cf/defog/sqlcoder-7b-2",
+                      "@hf/nexusflow/starling-lm-7b-beta",
+                      "@cf/tinyllama/tinyllama-1.1b-chat-v1.0",
+                      "@hf/thebloke/zephyr-7b-beta-awq",
+                      "@hf/thebloke/deepseek-coder-6.7b-base-awq"
+                    ]
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              }
+            }
+          }
+        },
         {
           "if": {
             "properties": {