fix(server): significantly increase the default limits (#159)

hyperonym · Apr 19, 2023 · 6dbf68b · 6dbf68b
1 parent 562cb50
commit 6dbf68b
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -36,8 +36,8 @@ ENV SERVER_CHANNEL_TIMEOUT="300"
 ENV SERVER_MODEL_NAME=""
 ENV SERVER_NO_PLAYGROUND="false"
 ENV SERVER_CORS_ORIGINS="*"
-ENV COMPLETION_MAX_PROMPT="4096"
-ENV COMPLETION_MAX_TOKENS="4096"
+ENV COMPLETION_MAX_PROMPT="32768"
+ENV COMPLETION_MAX_TOKENS="8192"
 ENV COMPLETION_MAX_N="5"
 ENV COMPLETION_MAX_LOGPROBS="5"
 ENV COMPLETION_MAX_INTERVAL="50"

diff --git a/basaran/__init__.py b/basaran/__init__.py
@@ -36,8 +36,8 @@ def is_true(value):
 SERVER_CORS_ORIGINS = os.getenv("SERVER_CORS_ORIGINS", "*")
 
 # Completion-related arguments:
-COMPLETION_MAX_PROMPT = int(os.getenv("COMPLETION_MAX_PROMPT", "4096"))
-COMPLETION_MAX_TOKENS = int(os.getenv("COMPLETION_MAX_TOKENS", "4096"))
+COMPLETION_MAX_PROMPT = int(os.getenv("COMPLETION_MAX_PROMPT", "32768"))
+COMPLETION_MAX_TOKENS = int(os.getenv("COMPLETION_MAX_TOKENS", "8192"))
 COMPLETION_MAX_N = int(os.getenv("COMPLETION_MAX_N", "5"))
 COMPLETION_MAX_LOGPROBS = int(os.getenv("COMPLETION_MAX_LOGPROBS", "5"))
 COMPLETION_MAX_INTERVAL = int(os.getenv("COMPLETION_MAX_INTERVAL", "50"))