Fix strip python type (#1602)

* update * the fix * Revert "update" This reverts commit 4c2f32f. * add a test and rebase * style * oups
huggingface · Aug 7, 2024 · 49dafd7 · 49dafd7
1 parent bded212
commit 49dafd7
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/bindings/python/src/normalizers.rs b/bindings/python/src/normalizers.rs
@@ -65,7 +65,7 @@ impl PyNormalizer {
                         Py::new(py, (PyBertNormalizer {}, base))?.into_py(py)
                     }
                     NormalizerWrapper::StripNormalizer(_) => {
-                        Py::new(py, (PyBertNormalizer {}, base))?.into_py(py)
+                        Py::new(py, (PyStrip {}, base))?.into_py(py)
                     }
                     NormalizerWrapper::Prepend(_) => Py::new(py, (PyPrepend {}, base))?.into_py(py),
                     NormalizerWrapper::ByteLevel(_) => {

diff --git a/bindings/python/tests/bindings/test_normalizers.py b/bindings/python/tests/bindings/test_normalizers.py
@@ -68,12 +68,13 @@ def test_can_make_sequences(self):
         assert output == "hello"
 
     def test_items(self):
-        normalizers = Sequence([BertNormalizer(True, True), Prepend()])
+        normalizers = Sequence([BertNormalizer(True, True), Prepend(), Strip()])
         assert normalizers[1].__class__ == Prepend
         normalizers[0].lowercase = False
         assert not normalizers[0].lowercase
+        assert normalizers[2].__class__ == Strip
         with pytest.raises(IndexError):
-            print(normalizers[2])
+            print(normalizers[3])
 
 
 class TestLowercase: