openrouter integration

timothelaborie · Jan 14, 2024 · 55e7de7 · 55e7de7
1 parent 973f62a
commit 55e7de7
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 71 deletions.
diff --git a/src/lib/ApiUtil.svelte b/src/lib/ApiUtil.svelte
@@ -1,6 +1,7 @@
 <script context="module" lang="ts">
   // This makes it possible to override the OpenAI API base URL in the .env file
-  const apiBase = import.meta.env.VITE_API_BASE || 'https://api.openai.com'
+  // const apiBase = import.meta.env.VITE_API_BASE || 'https://api.openai.com'
+  const apiBase = import.meta.env.VITE_API_BASE || 'https://openrouter.ai/api'
   const endpointCompletions = import.meta.env.VITE_ENDPOINT_COMPLETIONS || '/v1/chat/completions'
   const endpointGenerations = import.meta.env.VITE_ENDPOINT_GENERATIONS || '/v1/images/generations'
   const endpointModels = import.meta.env.VITE_ENDPOINT_MODELS || '/v1/models'

diff --git a/src/lib/ChatCompletionResponse.svelte b/src/lib/ChatCompletionResponse.svelte
@@ -167,7 +167,7 @@ export class ChatCompletionResponse {
   updateFromError (errorMessage: string): void {
     if (this.finished || this.error) return
     this.error = errorMessage
-    if (this.opts.autoAddMessages) {
+    if (this.opts.autoAddMessages && errorMessage != "Unexpected end of JSON input") {
       addMessage(this.chat.id, {
         role: 'error',
         content: `Error: ${errorMessage}`,

diff --git a/src/lib/providers/openai/models.svelte b/src/lib/providers/openai/models.svelte
@@ -1,13 +1,45 @@
 <script context="module" lang="ts">
-    import { getApiBase, getEndpointCompletions, getEndpointGenerations } from '../../ApiUtil.svelte'
-    import { countTokens } from '../../Models.svelte'
-    import { countMessageTokens } from '../../Stats.svelte'
-    import { globalStorage } from '../../Storage.svelte'
-    import type { Chat, Message, Model, ModelDetail } from '../../Types.svelte'
-    import { chatRequest, imageRequest } from './request.svelte'
-    import { checkModel } from './util.svelte'
-    import { encode } from 'gpt-tokenizer'
-    import { get } from 'svelte/store'
+
+
+
+import { getApiBase, getEndpointCompletions, getEndpointGenerations } from '../../ApiUtil.svelte'
+import { countTokens } from '../../Models.svelte'
+import { countMessageTokens } from '../../Stats.svelte'
+import { globalStorage } from '../../Storage.svelte'
+import type { Chat, Message, Model, ModelDetail } from '../../Types.svelte'
+import { chatRequest, imageRequest } from './request.svelte'
+import { checkModel } from './util.svelte'
+import { encode } from 'gpt-tokenizer'
+import { get } from 'svelte/store'
+
+let modelsList = {
+"data": [
+{
+"id": "nousresearch/nous-capybara-34b",
+"name": "Nous: Capybara 34B",
+"description": "This model is trained on the Yi-34B model for 3 epochs on the Capybara dataset. It's the first 34B Nous model and first 200K context length Nous model.\n\n**Note:** This endpoint currently supports 32k context.",
+"pricing": {
+"prompt": "0.0000007",
+"completion": "0.0000028"
+},
+"context_length": 32768,
+"architecture": {
+"tokenizer": "Llama2",
+"instruct_type": "vicuna"
+},
+"top_provider": {
+"max_completion_tokens": null
+},
+"per_request_limits": {
+"prompt_tokens": "5301093",
+"completion_tokens": "1325273"
+}
+}
+]
+}
+
+
+
 
 const hiddenSettings = {
       startSequence: true,
@@ -56,54 +88,44 @@ const chatModelBase = {
   }
 } as ModelDetail
 
-// Reference: https://openai.com/pricing#language-models
-const gpt35 = {
-      ...chatModelBase,
-      prompt: 0.0000015, // $0.0015 per 1000 tokens prompt
-      completion: 0.000002, // $0.002 per 1000 tokens completion
-      max: 4096 // 4k max token buffer
-}
-const gpt3516k = {
-      ...chatModelBase,
-      prompt: 0.000001, // $0.001 per 1000 tokens prompt
-      completion: 0.0000015, // $0.0015 per 1000 tokens completion
-      max: 16384 // 16k max token buffer
-}
-const gpt4 = {
-      ...chatModelBase,
-      prompt: 0.00003, // $0.03 per 1000 tokens prompt
-      completion: 0.00006, // $0.06 per 1000 tokens completion
-      max: 8192 // 8k max token buffer
-}
-const gpt432k = {
-      ...chatModelBase,
-      prompt: 0.00006, // $0.06 per 1000 tokens prompt
-      completion: 0.00012, // $0.12 per 1000 tokens completion
-      max: 32768 // 32k max token buffer
+
+
+function fetchModelsList() {
+  const request = new XMLHttpRequest();
+  request.open('GET', 'https://openrouter.ai/api/v1/models', false); // false for synchronous request
+  request.send(null);
+
+  if (request.status === 200) {
+    const data = JSON.parse(request.responseText);
+    modelsList.data = data.data;
+    console.log("fetched models list");
+  } else {
+    throw new Error(`HTTP error! status: ${request.status}`);
+  }
 }
-const gpt4128kpreview = {
+
+
+function generateModels(modelsList: any): any {
+  console.log("generateModels");
+  fetchModelsList();
+  const models: any = {};
+
+  modelsList.data.forEach((model: any) => {
+    models[model.id] = {
       ...chatModelBase,
-      prompt: 0.00001, // $0.01 per 1000 tokens prompt
-      completion: 0.00003, // $0.03 per 1000 tokens completion
-      max: 131072 // 128k max token buffer
-}
+      prompt: parseFloat(model.pricing.prompt),
+      completion: parseFloat(model.pricing.completion),
+      max: model.context_length
+    };
+  });
 
-export const chatModels : Record<string, ModelDetail> = {
-  'gpt-3.5-turbo': { ...gpt3516k },
-  'gpt-3.5-turbo-0301': { ...gpt35 },
-  'gpt-3.5-turbo-0613': { ...gpt35 },
-  'gpt-3.5-turbo-1106': { ...gpt3516k },
-  'gpt-3.5-turbo-16k': { ...gpt3516k },
-  'gpt-3.5-turbo-16k-0613': { ...gpt3516k },
-  'gpt-4': { ...gpt4 },
-  'gpt-4-0314': { ...gpt4 },
-  'gpt-4-0613': { ...gpt4 },
-  'gpt-4-1106-preview': { ...gpt4128kpreview },
-  'gpt-4-32k': { ...gpt432k },
-  'gpt-4-32k-0314': { ...gpt432k },
-  'gpt-4-32k-0613': { ...gpt432k }
+  return models;
 }
 
+export const chatModels : Record<string, ModelDetail> = generateModels(modelsList);
+
+
+
 const imageModelBase = {
   type: 'image',
   prompt: 0.00,
@@ -197,4 +219,5 @@ export const imageModels : Record<string, ModelDetail> = {
       }
 }
 
-</script>
+</script>
+
diff --git a/src/lib/providers/openai/request.svelte b/src/lib/providers/openai/request.svelte
@@ -52,9 +52,14 @@ export const chatRequest = async (
               if (ev.data === '[DONE]') {
               // ?? anything to do when "[DONE]"?
               } else {
-                const data = JSON.parse(ev.data)
-                // console.log('data', data)
-                window.setTimeout(() => { chatResponse.updateFromAsyncResponse(data) }, 1)
+                try{
+                  const data = JSON.parse(ev.data)
+                  // console.log('data', data)
+                  window.setTimeout(() => { chatResponse.updateFromAsyncResponse(data) }, 1)
+                }
+                catch(e){
+                  // console.error(e)
+                }
               }
             }
           },

diff --git a/src/lib/providers/openai/util.svelte b/src/lib/providers/openai/util.svelte
@@ -25,19 +25,21 @@ const getSupportedModels = async (): Promise<Record<string, boolean>> => {
   const openAiKey = get(apiKeyStorage)
   if (!openAiKey) return {}
   try {
-        const result = (await (
-          await fetch(getApiBase() + getEndpointModels(), {
-            method: 'GET',
-            headers: {
-              Authorization: `Bearer ${openAiKey}`,
-              'Content-Type': 'application/json'
-            }
-          })
-        ).json()) as ResponseModels
-        availableModels = result.data.reduce((a, v) => {
-          a[v.id] = v
-          return a
-        }, {})
+        // const result = (await (
+        //   await fetch(getApiBase() + getEndpointModels(), {
+        //     method: 'GET',
+        //     headers: {
+        //       Authorization: `Bearer ${openAiKey}`,
+        //       'Content-Type': 'application/json'
+        //     }
+        //   })
+        // ).json()) as ResponseModels
+        // availableModels = result.data.reduce((a, v) => {
+        //   console.log('v', v)
+        //   a[v.id] = v
+        //   return a
+        // }, {})
+        availableModels = []
         return availableModels
   } catch (e) {
         availableModels = {}
@@ -51,6 +53,7 @@ export const checkModel = async (modelDetail: ModelDetail) => {
   const supportedModels = await getSupportedModels()
   if (modelDetail.type === 'chat' || modelDetail.type === 'instruct') {
         modelDetail.enabled = !!supportedModels[modelDetail.modelQuery || '']
+        modelDetail.enabled = true
   } else {
         // image request.  If we have any models, allow image endpoint
         modelDetail.enabled = !!Object.keys(supportedModels).length