From b151ff56969b7d352917b1a1f7be2c6ab4561a8e Mon Sep 17 00:00:00 2001
From: Joshua Lochner <admin@xenova.com>
Date: Tue, 5 Mar 2024 00:03:00 +0200
Subject: [PATCH] Add claude and mistral tokenizers

---
 examples/tokenizer-playground/src/App.jsx | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/tokenizer-playground/src/App.jsx b/examples/tokenizer-playground/src/App.jsx
index c6546d20f..9105e0f20 100644
--- a/examples/tokenizer-playground/src/App.jsx
+++ b/examples/tokenizer-playground/src/App.jsx
@@ -70,6 +70,8 @@ function App() {
           <option value="Xenova/gpt-4">gpt-4 / gpt-3.5-turbo / text-embedding-ada-002</option>
           <option value="Xenova/text-davinci-003">text-davinci-003 / text-davinci-002</option>
           <option value="Xenova/gpt-3">gpt-3</option>
+          <option value="Xenova/claude-tokenizer">Claude 3</option>
+          <option value="Xenova/mistral-tokenizer">Mistral</option>
           <option value="Xenova/gemma-tokenizer">Gemma</option>
           <option value="Xenova/llama-tokenizer">LLaMA / Llama 2</option>
           <option value="Xenova/t5-small">T5</option>