cleanup warning, add assert llama.cpp pre-tokenizer fails test

outlines/integrations/llamacpp.py

@@ -54,7 +54,7 @@ def __init__(self, model: "Llama"):
 
         self.decode = tokenizer.decode
 
-        # ### TODO: Remove when https://github.com/ggerganov/llama.cpp/pull/5613 is resolved
+        # TODO: Remove when https://github.com/ggerganov/llama.cpp/pull/5613 is resolved
         try:
             self.vocabulary = model.tokenizer_.hf_tokenizer.get_vocab()
         except AttributeError:

tests/generate/test_integration_llamacpp.py

@@ -269,3 +269,14 @@ def test_byte_tokenizer_regression(repo, model_path, hf_tokenizer_uri):
     )
     generator = generate.choice(model, ["skirt", "dress", "pen", "jacket"])
     generator("Pick the odd word out: skirt, dress, pen, jacket")
+
+
+def test_llama_cpp_pre_tokenizer_remains_broken():
+    """If fails, llama.cpp pre-tokenizer is fixed -> revert #892, remove `with pytest.raises`"""
+    repo = "Qwen/Qwen1.5-0.5B-Chat-GGUF"
+    model_path = "*q2*.gguf"
+
+    model = models.llamacpp(repo, model_path)
+    generator = generate.choice(model, ["skirt", "dress", "pen", "jacket"])
+    with pytest.raises(RuntimeError):
+        generator("Pick the odd word out: skirt, dress, pen, jacket")