optional max_tokens

ChatGPTNextWeb · Mar 27, 2024 · 8eb78c9 · 8eb78c9
1 parent 43824bd
commit 8eb78c9
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 15 deletions.
diff --git a/app/client/api.ts b/app/client/api.ts
@@ -34,6 +34,7 @@ export interface LLMConfig {
   stream?: boolean;
   presence_penalty?: number;
   frequency_penalty?: number;
+  useMaxTokens?: boolean;
 }
 
 export interface ChatOptions {

diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts
@@ -106,8 +106,7 @@ export class ChatGPTApi implements LLMApi {
       presence_penalty: modelConfig.presence_penalty,
       frequency_penalty: modelConfig.frequency_penalty,
       top_p: modelConfig.top_p,
-      // max_tokens: Math.max(modelConfig.max_tokens, 1024),
-      // Please do not ask me why not send max_tokens, no reason, this param is just shit, I dont want to explain anymore.
+      max_tokens: modelConfig.useMaxTokens ? modelConfig.max_tokens : undefined,
     };
 
     // add max_tokens to vision model

diff --git a/app/components/model-config.tsx b/app/components/model-config.tsx
@@ -72,25 +72,38 @@ export function ModelConfigList(props: {
           }}
         ></InputRange>
       </ListItem>
-      <ListItem
-        title={Locale.Settings.MaxTokens.Title}
-        subTitle={Locale.Settings.MaxTokens.SubTitle}
-      >
+      <ListItem title={"Use Max Tokens"}>
         <input
-          type="number"
-          min={1024}
-          max={512000}
-          value={props.modelConfig.max_tokens}
+          type="checkbox"
+          checked={props.modelConfig.useMaxTokens}
           onChange={(e) =>
             props.updateConfig(
-              (config) =>
-                (config.max_tokens = ModalConfigValidator.max_tokens(
-                  e.currentTarget.valueAsNumber,
-                )),
+              (config) => (config.useMaxTokens = e.currentTarget.checked),
             )
           }
         ></input>
       </ListItem>
+      {props.modelConfig.useMaxTokens && (
+        <ListItem
+          title={Locale.Settings.MaxTokens.Title}
+          subTitle={Locale.Settings.MaxTokens.SubTitle}
+        >
+          <input
+            type="number"
+            min={1}
+            max={32768}
+            value={props.modelConfig.max_tokens}
+            onChange={(e) =>
+              props.updateConfig(
+                (config) =>
+                  (config.max_tokens = ModalConfigValidator.max_tokens(
+                    e.currentTarget.valueAsNumber,
+                  )),
+              )
+            }
+          ></input>
+        </ListItem>
+      )}
 
       {props.modelConfig.model.startsWith("gemini") ? null : (
         <>

diff --git a/app/store/config.ts b/app/store/config.ts
@@ -49,14 +49,15 @@ export const DEFAULT_CONFIG = {
     model: "gpt-3.5-turbo" as ModelType,
     temperature: 0.5,
     top_p: 1,
-    max_tokens: 4000,
+    max_tokens: 2048,
     presence_penalty: 0,
     frequency_penalty: 0,
     sendMemory: true,
     historyMessageCount: 4,
     compressMessageLengthThreshold: 1000,
     enableInjectSystemPrompts: true,
     template: DEFAULT_INPUT_TEMPLATE,
+    useMaxTokens: false,
   },
 };