Use ungated version of mistral tokenizer (#718)

xenova · Apr 20, 2024 · 1a9964f · 1a9964f
1 parent 992f643
commit 1a9964f
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -2986,7 +2986,7 @@ export class PreTrainedTokenizer extends Callable {
      * ```javascript
      * import { AutoTokenizer } from "@xenova/transformers";
      * 
-     * const tokenizer = await AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1");
+     * const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1");
      * 
      * const chat = [
      *   { "role": "user", "content": "Hello, how are you?" },

diff --git a/tests/generate_tests.py b/tests/generate_tests.py
@@ -229,7 +229,7 @@
         'basic',
     ],
 
-    'mistralai/Mistral-7B-Instruct-v0.1': [
+    'Xenova/mistral-tokenizer-v1': [
         'basic',
     ],
 

diff --git a/tests/tokenizers.test.js b/tests/tokenizers.test.js
@@ -334,7 +334,7 @@ describe('Extra decoding tests', () => {
 
 describe('Chat templates', () => {
     it('should generate a chat template', async () => {
-        const tokenizer = await AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1");
+        const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1");
 
         const chat = [
             { "role": "user", "content": "Hello, how are you?" },