From 05e983d6025b331e0ab0f5eb66a6b001618638ad Mon Sep 17 00:00:00 2001
From: Pedro Cuenca <pedro@huggingface.co>
Date: Wed, 10 Sep 2025 15:29:02 +0200
Subject: [PATCH] Remove old comment.

specialTokens are now conditionally ignored on skipSpecialTokens at
decode time.
---
 Sources/Tokenizers/Tokenizer.swift | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Sources/Tokenizers/Tokenizer.swift b/Sources/Tokenizers/Tokenizer.swift
index b410f3e5..f8662c34 100644
--- a/Sources/Tokenizers/Tokenizer.swift
+++ b/Sources/Tokenizers/Tokenizer.swift
@@ -321,7 +321,6 @@ public class PreTrainedTokenizer: Tokenizer {
         }.joined(separator: "|")
         addedTokensRegex = try? NSRegularExpression(pattern: addedTokensRegexString, options: [])
 
-        // TODO: specialTokens are stored but never used
         self.specialTokens = specialTokens
         self.addedTokens = Set(addedTokens.keys)