From 2df14fc8131ffd4e7c2749cd2a42a45e089dc238 Mon Sep 17 00:00:00 2001
From: Joshua Lochner <admin@xenova.com>
Date: Fri, 12 Jan 2024 02:28:08 +0200
Subject: [PATCH] Update tokenizers.js

---
 src/tokenizers.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tokenizers.js b/src/tokenizers.js
index a786cd1ed..030acfec7 100644
--- a/src/tokenizers.js
+++ b/src/tokenizers.js
@@ -2463,7 +2463,7 @@ export class PreTrainedTokenizer extends Callable {
         this.sep_token = this.getToken('sep_token');
         this.sep_token_id = this.model.tokens_to_ids.get(this.sep_token);
 
-        this.unk_token = this.getToken(tokenizerConfig, 'unk_token');
+        this.unk_token = this.getToken('unk_token');
         this.unk_token_id = this.model.tokens_to_ids.get(this.unk_token);
 
         this.model_max_length = tokenizerConfig.model_max_length;