From 52f469907e27cff3567dd39f88db0b94cae1e093 Mon Sep 17 00:00:00 2001
From: NaveenMittal0 <144010273+NaveenMittal0@users.noreply.github.com>
Date: Mon, 24 Feb 2025 11:37:05 +0530
Subject: [PATCH] updating tokenizer version from 0.20.0 to 0.21.0

---
 rust/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 0758441..f875fac 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -8,6 +8,6 @@ crate-type = ["staticlib"]
 
 [dependencies]
 
-tokenizers = { version = "0.20.0", default-features = false, features = ["onig"] }
+tokenizers = { version = "0.21.0", default-features = false, features = ["onig"] }
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1.0"