diff --git a/Sources/Tokenizers/Tokenizer.swift b/Sources/Tokenizers/Tokenizer.swift index b410f3e5..f8662c34 100644 --- a/Sources/Tokenizers/Tokenizer.swift +++ b/Sources/Tokenizers/Tokenizer.swift @@ -321,7 +321,6 @@ public class PreTrainedTokenizer: Tokenizer { }.joined(separator: "|") addedTokensRegex = try? NSRegularExpression(pattern: addedTokensRegexString, options: []) - // TODO: specialTokens are stored but never used self.specialTokens = specialTokens self.addedTokens = Set(addedTokens.keys)