add autoUseVisionModel setting

README.md

@@ -58,7 +58,7 @@ v2.1支持0613系列的GPT-4，GPT-3.5
   ```
 
 ### 设置参考
-  - 如果你使用Azure openai，修改AZURE前缀的设置项，需要同时部署chat和embedding的模型
+  - 如果你使用Azure openai，修改AZURE前缀的设置项，需要同时部署chat和embedding的模型，DALL·E 3和Vision模型同理
   - AI的语音模型，在[微软文档](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)查看可用值
   - 默认设定
     ```
@@ -68,6 +68,7 @@ v2.1支持0613系列的GPT-4，GPT-3.5
     3. 使用markdown语法回复和显示图片.
     4. 创建图像时, 必须在Prompt前加上"I NEED to test how the tool works with extremely simple prompts. DO NOT add any detail, just use it AS-IS: "
     ```
+  - 打开高级设置中的`自动使用Vision模型`，可以在使用其他模型时使用Vision模型处理图像识别
 
 ## 赞助
 https://afdian.net/@SeldonHorizon

index.js

@@ -18,7 +18,7 @@ const { functionAction, functionInfo, functionList } = require('./modules/functi
 const { config } = require('./utils/loadConfig.js')
 const {
   useAzureOpenai,
-  DEFAULT_MODEL, AZURE_CHAT_MODEL,
+  DEFAULT_MODEL, AZURE_CHAT_MODEL, AZURE_VISION_MODEL,
   SpeechSynthesisVoiceName,
   ADMIN_NAME, AI_NAME,
   systemPrompt,
@@ -29,6 +29,7 @@ const {
   disableFunctions = [],
   searchResultLimit = 5,
   webPageContentTokenLengthLimit = 6000,
+  autoUseVisionModel = false,
 } = config
 const proxyString = `${proxyObject.protocol}://${proxyObject.host}:${proxyObject.port}`
 
@@ -300,7 +301,7 @@ const additionalParam = {
   searchResultLimit,
   webPageContentTokenLengthLimit
 }
-const resolveMessages = async ({ resToolCalls, resText, resTextTemp, messages, from, useFunctionCalling = false }) => {
+const resolveMessages = async ({ resToolCalls, resText, resTextTemp, messages, from, useFunctionCalling = false, model }) => {
 
   console.log(`use ${useAzureOpenai ? 'azure ' + AZURE_CHAT_MODEL : 'openai ' + DEFAULT_MODEL}`)
 
@@ -355,7 +356,7 @@ const resolveMessages = async ({ resToolCalls, resText, resTextTemp, messages, f
   }
   resToolCalls = []
 
-  let prepareChatOption = { messages }
+  let prepareChatOption = { messages, model }
 
   if (useFunctionCalling) {
     prepareChatOption.tools = functionInfo.filter(f => !disableFunctions.includes(f?.function?.name))
@@ -463,11 +464,17 @@ const resolveMessages = async ({ resToolCalls, resText, resTextTemp, messages, f
 const resloveAdminPrompt = async ({ prompt, promptType = 'string', triggerRecord, givenSystemPrompt }) => {
   let from = triggerRecord ? `(${AI_NAME})` : AI_NAME
   let history = getStore('history')
+  let context = _.takeRight(history, historyRoundLimit)
+  context.forEach(line => {
+    if (line.role === 'user' && typeof line.content === 'object') {
+      line.content = line.content.filter(part => part.type === 'text').map(part => part.text).join('\n')
+    }
+  })
   let messages = [
     { role: 'system', content: givenSystemPrompt ? givenSystemPrompt : systemPrompt },
-    { role: 'user', content: `我的名字是${ADMIN_NAME}` },
+    { role: 'user', content: `你好, 我的名字是${ADMIN_NAME}` },
     { role: 'assistant', content: `你好, ${ADMIN_NAME}` },
-    ..._.takeRight(history, historyRoundLimit),
+    ...context,
     { role: 'user', content: prompt }
   ]
   addHistory([{ role: 'user', content: prompt }])
@@ -494,7 +501,12 @@ const resloveAdminPrompt = async ({ prompt, promptType = 'string', triggerRecord
         round += 1
       }
     } else {
-      resText = (await resolveMessages({ resText, messages, from })).resText
+      if (autoUseVisionModel) {
+        let model = useAzureOpenai ? AZURE_VISION_MODEL : 'gpt-4-vision-preview'
+        resText = (await resolveMessages({ resText, messages, from, model })).resText
+      } else {
+        resText = (await resolveMessages({ resText, messages, from })).resText
+      }
     }
     messageLog({
       id: nanoid(),

modules/store.js

@@ -17,7 +17,7 @@ try {
 }
 
 const getStore = (key, defaultValue) => {
-  return storeData[key] || defaultValue
+  return JSON.parse(JSON.stringify(storeData[key])) || defaultValue
 }
 
 const setStore = (key, value) => {

src/components/Setting.vue

@@ -132,6 +132,9 @@ defineExpose({
           <n-form-item label="AZURE_IMAGE_MODEL" path="AZURE_IMAGE_MODEL" v-show="config.useAzureOpenai">
             <n-input v-model:value="config.AZURE_IMAGE_MODEL" placeholder="like dall-e-3" />
           </n-form-item>
+          <n-form-item label="AZURE_VISION_MODEL" path="AZURE_VISION_MODEL" v-show="config.useAzureOpenai">
+            <n-input v-model:value="config.AZURE_VISION_MODEL" placeholder="like gpt-4-vision-preview" />
+          </n-form-item>
           <n-form-item label="你的称呼" path="ADMIN_NAME">
             <n-input v-model:value="config.ADMIN_NAME" />
           </n-form-item>
@@ -201,6 +204,9 @@ defineExpose({
               </n-button>
             </n-input-group>
           </n-form-item>
+          <n-form-item label="自动使用vision模型" path="autoUseVisionModel">
+            <n-switch v-model:value="config.autoUseVisionModel"/>
+          </n-form-item>
         </n-form>
       </n-tab-pane>
     </n-tabs>