README

README.md

@@ -1,8 +1,20 @@
 # openai api mock
 
+this project aims to convert / proxy Cloudflare Workers AI responses to OpenAI API compatible responses so Workers AI models can be used with any OpenAI / ChatGPT compatible client
+
+## installation
+
+tba
+
+## features
+
+- supports streaming and non-streaming responses
+- rewrites the different models 'gpt-3' and 'gpt-4' to use `@cf/meta/llama-2-7b-chat-fp16`
+- if the openAI client can be configured to provide other model names, just put in the cloudflare model id instead of gpt-4
+
 ## use with llm
 
-- [this is llm](https://llm.datasette.io/)
+- [read and install everything about llm here](https://llm.datasette.io/)
 
 create this file `~/Library/Application\ Support/io.datasette.llm/extra-openai-models.yaml`:
 
@@ -12,7 +24,13 @@ create this file `~/Library/Application\ Support/io.datasette.llm/extra-openai-m
   api_base: 'https://openai-api.foobar.workers.dev/'
 ```
 
-use it:
+use it with streaming (recommended):
+
+```shell
+llm chat -m cloudflare
+```
+
+use it without streaming:
 
 ```shell
 llm chat --no-stream -m cloudflare

src/index.js

@@ -93,7 +93,7 @@ export default {
     for await (const part of resp) {
       const text = textDecoder.decode(part)
       const lines = text.split('\n')
-      lines.forEach((element, index) => {
+      lines.forEach((element, idx) => {
         if (element.includes('[DONE]')) {
           const data = {
             id: 'chatcmpl-123',
@@ -107,6 +107,7 @@ export default {
         if (element.startsWith('data: ') && element.endsWith('}')) {
           const out = element.replace('data: ', '').trim()
           const json = JSON.parse(out)
+          console.log(`index: ${idx}, json: ${json.response}`)
           const data = {
             id: 'chatcmpl-123',
             object: 'chat.completion.chunk',
@@ -121,9 +122,9 @@ export default {
               },
             ],
           }
-          if (index === 0 || index === 2) {
-            data.choices[0].delta.content = json.response.trim()
-          }
+          // if (idx === 0 || idx === 2) {
+          //   data.choices[0].delta.content = json.response.trim()
+          // }
           writer.write(textEncoder.encode(`data: ${JSON.stringify(data)}\n\n`))
         }
       })

wrangler.toml

@@ -0,0 +1,99 @@
+name = "openai-api-mock"
+main = "src/index.js"
+compatibility_date = "2024-04-04"
+compatibility_flags = ["nodejs_compat"]
+logpush = true
+
+# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
+# Note: Use secrets to store sensitive data.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables
+# [vars]
+# MY_VARIABLE = "production_value"
+
+# Bind the Workers AI model catalog. Run machine learning models, powered by serverless GPUs, on Cloudflare’s global network
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#workers-ai
+[ai]
+binding = "AI"
+
+# Bind an Analytics Engine dataset. Use Analytics Engine to write analytics within your Pages Function.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#analytics-engine-datasets
+# [[analytics_engine_datasets]]
+# binding = "MY_DATASET"
+
+# Bind a headless browser instance running on Cloudflare's global network.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#browser-rendering
+# [browser]
+# binding = "MY_BROWSER"
+
+# Bind a D1 database. D1 is Cloudflare’s native serverless SQL database.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#d1-databases
+# [[d1_databases]]
+# binding = "MY_DB"
+# database_name = "my-database"
+# database_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+
+# Bind a dispatch namespace. Use Workers for Platforms to deploy serverless functions programmatically on behalf of your customers.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#dispatch-namespace-bindings-workers-for-platforms
+# [[dispatch_namespaces]]
+# binding = "MY_DISPATCHER"
+# namespace = "my-namespace"
+
+# Bind a Durable Object. Durable objects are a scale-to-zero compute primitive based on the actor model.
+# Durable Objects can live for as long as needed. Use these when you need a long-running "server", such as in realtime apps.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#durable-objects
+# [[durable_objects.bindings]]
+# name = "MY_DURABLE_OBJECT"
+# class_name = "MyDurableObject"
+
+# Durable Object migrations.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#migrations
+# [[migrations]]
+# tag = "v1"
+# new_classes = ["MyDurableObject"]
+
+# Bind a Hyperdrive configuration. Use to accelerate access to your existing databases from Cloudflare Workers.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#hyperdrive
+# [[hyperdrive]]
+# binding = "MY_HYPERDRIVE"
+# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+# Bind a KV Namespace. Use KV as persistent storage for small key-value pairs.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#kv-namespaces
+# [[kv_namespaces]]
+# binding = "MY_KV_NAMESPACE"
+# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+# Bind an mTLS certificate. Use to present a client certificate when communicating with another service.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#mtls-certificates
+# [[mtls_certificates]]
+# binding = "MY_CERTIFICATE"
+# certificate_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+
+# Bind a Queue producer. Use this binding to schedule an arbitrary task that may be processed later by a Queue consumer.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
+# [[queues.producers]]
+# binding = "MY_QUEUE"
+# queue = "my-queue"
+
+# Bind a Queue consumer. Queue Consumers can retrieve tasks scheduled by Producers to act on them.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
+# [[queues.consumers]]
+# queue = "my-queue"
+
+# Bind an R2 Bucket. Use R2 to store arbitrarily large blobs of data, such as files.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#r2-buckets
+# [[r2_buckets]]
+# binding = "MY_BUCKET"
+# bucket_name = "my-bucket"
+
+# Bind another Worker service. Use this binding to call another Worker without network overhead.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings
+# [[services]]
+# binding = "MY_SERVICE"
+# service = "my-service"
+
+# Bind a Vectorize index. Use to store and query vector embeddings for semantic search, classification and other vector search use-cases.
+# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#vectorize-indexes
+# [[vectorize]]
+# binding = "MY_INDEX"
+# index_name = "my-index"