fix: add a method to detect vision model

README.md

@@ -233,6 +233,11 @@ To control custom models, use `+` to add a custom model, use `-` to hide a model
 
 User `-all` to disable all default models, `+all` to enable all default models.
 
+### `CUSTOM_VISION_MODELS`（可选）
+
+> Example: `+claude-3-opus-20240229,+claude-3-sonnet-20240229` means add `claude-3-opus-20240229` and `claude-3-sonnet-20240229` to vision model list, use `+` to add a custom model, separated modols by comma.
+
+
 ## Requirements
 
 NodeJS >= 18, Docker >= 20

README_CN.md

@@ -137,6 +137,10 @@ Google Gemini Pro Api Url.
 
 用来控制模型列表，使用 `+` 增加一个模型，使用 `-` 来隐藏一个模型，使用 `模型名=展示名` 来自定义模型的展示名，用英文逗号隔开。
 
+### `CUSTOM_VISION_MODELS`（可选）
+
+> 示例：`+claude-3-opus-20240229,+claude-3-sonnet-20240229` 表示增加 `claude-3-opus-20240229` 和 `claude-3-sonnet-20240229` 到支持视觉模型的列表，使用 `+` 增加一个模型，用英文逗号隔开。
+
 ## 开发
 
 点击下方按钮，开始二次开发：

app/api/config/route.ts

@@ -13,6 +13,7 @@ const DANGER_CONFIG = {
   hideBalanceQuery: serverConfig.hideBalanceQuery,
   disableFastLink: serverConfig.disableFastLink,
   customModels: serverConfig.customModels,
+  customVisionModels: serverConfig.customVisionModels,
 };
 
 declare global {

app/config/server.ts

@@ -21,6 +21,7 @@ declare global {
       ENABLE_BALANCE_QUERY?: string; // allow user to query balance or not
       DISABLE_FAST_LINK?: string; // disallow parse settings from url or not
       CUSTOM_MODELS?: string; // to control custom models
+      CUSTOM_VISION_MODELS?: string; // allow user append custom vision models
 
       // azure only
       AZURE_URL?: string; // https://{azure-url}/openai/deployments/{deploy-name}
@@ -59,14 +60,15 @@ export const getServerSideConfig = () => {
 
   const disableGPT4 = !!process.env.DISABLE_GPT4;
   let customModels = process.env.CUSTOM_MODELS ?? "";
+  let customVisionModels = process.env.CUSTOM_VISION_MODELS ?? "";
 
   if (disableGPT4) {
     if (customModels) customModels += ",";
     customModels += DEFAULT_MODELS.filter((m) => m.name.startsWith("gpt-4"))
       .map((m) => "-" + m.name)
       .join(",");
   }
-
+  if (customVisionModels) customVisionModels += ",";
   const isAzure = !!process.env.AZURE_URL;
   const isGoogle = !!process.env.GOOGLE_API_KEY;
 
@@ -106,5 +108,6 @@ export const getServerSideConfig = () => {
     hideBalanceQuery: !process.env.ENABLE_BALANCE_QUERY,
     disableFastLink: !!process.env.DISABLE_FAST_LINK,
     customModels,
+    customVisionModels,
   };
 };

app/store/access.ts

@@ -43,6 +43,7 @@ const DEFAULT_ACCESS_STATE = {
   disableGPT4: false,
   disableFastLink: false,
   customModels: "",
+  customVisionModels: "",
 };
 
 export const useAccessStore = createPersistStore(

app/utils.ts

@@ -3,6 +3,7 @@ import { showToast } from "./components/ui-lib";
 import Locale from "./locales";
 import { RequestMessage } from "./client/api";
 import { DEFAULT_MODELS } from "./constant";
+import { useAccessStore } from "@/app/store";
 
 export function trimTopic(topic: string) {
   // Fix an issue where double quotes still show in the Indonesian language
@@ -292,9 +293,10 @@ export function getMessageImages(message: RequestMessage): string[] {
 }
 
 export function isVisionModel(model: string) {
+  const accessStore = useAccessStore.getState();
   return (
     // model.startsWith("gpt-4-vision") ||
     // model.startsWith("gemini-pro-vision") ||
-    model.includes("vision")
+    model.includes("vision") || accessStore.customVisionModels.includes(model)
   );
 }