[tokenizers] Updates tokenizer to 0.19.1 (#3143)

deepjavalibrary · May 2, 2024 · 2287609 · 2287609
1 parent 00c78f6
commit 2287609
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/extensions/tokenizers/src/main/java/ai/djl/engine/rust/zoo/package-info.java b/extensions/tokenizers/src/main/java/ai/djl/engine/rust/zoo/package-info.java
@@ -10,5 +10,5 @@
  * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
  * and limitations under the License.
  */
-/** Contains the built-in {@link ai.djl.huggingface.zoo.RsModelZoo}. */
+/** Contains the built-in {@link ai.djl.engine.rust.zoo.RsModelZoo}. */
 package ai.djl.engine.rust.zoo;
diff --git a/gradle.properties b/gradle.properties
@@ -18,7 +18,7 @@ tensorflow_version=2.10.1
 trt_version=8.4.1
 onnxruntime_version=1.17.1
 sentencepiece_version=0.1.97
-tokenizers_version=0.15.2
+tokenizers_version=0.19.1
 llamacpp_version=b1696
 fasttext_version=0.9.2
 xgboost_version=2.0.3