fix memory leak in BPE

[tmm1]

turns out this isn't much of a cache. just a memory leak in disguise.

> let {LlamaTokenizer} = await import('./src/transformers.js')
> var tok = await LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v0.4')
> tok.encode("very long user input ".repeat(10))
[
     1, 1407, 1472, 1404, 1881,  1407,
  1472, 1404, 1881, 1407, 1472,  1404,
  1881, 1407, 1472, 1404, 1881,  1407,
  1472, 1404, 1881, 1407, 1472,  1404,
  1881, 1407, 1472, 1404, 1881,  1407,
  1472, 1404, 1881, 1407, 1472,  1404,
  1881, 1407, 1472, 1404, 1881, 29871
]
> tok.model.cache
Map(1) {
  '▁very▁long▁user▁input▁very▁long▁user▁input▁very▁long▁user▁input▁very▁long▁user▁input▁very▁long▁user▁input▁very▁long▁user▁input▁very▁long▁user▁input▁very▁long▁user▁input▁very▁long▁user▁input▁very▁long▁user▁input▁' => [
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁very', '▁long', '▁user', '▁input',
    '▁'
  ]
}

[xenova]

Hi there 👋 Indeed, this BPE cache is more useful to tokenizers that use a GPT-like pretokenizer:

import { AutoTokenizer } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.2';
    
const tok = await AutoTokenizer.from_pretrained('Xenova/gpt-4');
tok.encode("very long user input ".repeat(10));
console.log(tok.model.cache);

A similar note is present here:

https://github.com/huggingface/transformers/blob/13493215abceafc1653af88b045120014fb4c1fc/src/transformers/models/qwen2/tokenization_qwen2.py#L186-L190

However, it might be worth disabling it for LlamaTokenizer (and similar) or defining a max input length/max size/lru cache).