Allow use of gpt-2 and gpt-3.5 in encoding_for_model (#185)

openai · Feb 9, 2024 · 84d88dc · 84d88dc
1 parent 01df436
commit 84d88dc
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/tiktoken/model.py b/tiktoken/model.py
@@ -20,6 +20,7 @@
     # chat
     "gpt-4": "cl100k_base",
     "gpt-3.5-turbo": "cl100k_base",
+    "gpt-3.5": "cl100k_base",  # Common shorthand
     "gpt-35-turbo": "cl100k_base",  # Azure deployment name
     # base
     "davinci-002": "cl100k_base",
@@ -63,6 +64,7 @@
     "code-search-ada-code-001": "r50k_base",
     # open source
     "gpt2": "gpt2",
+    "gpt-2": "gpt2",  # Maintains consistency with gpt-4
 }