test: update tests

src/rubrix/server/apis/v0/models/token_classification.py

@@ -122,6 +122,7 @@ def __init__(self, **data):
         super().__init__(**data)
 
         self.__chars2tokens__, self.__tokens2chars__ = self.__build_indices_map__()
+
         self.check_annotation(self.prediction)
         self.check_annotation(self.annotation)
 
@@ -162,7 +163,6 @@ def chars2tokens_index():
             chars_map = {}
             current_token = 0
             current_token_char_start = 0
-
             for idx, char in enumerate(self.text):
                 relative_idx = idx - current_token_char_start
                 if (
@@ -201,21 +201,8 @@ def check_annotation(
         annotation: Optional[TokenClassificationAnnotation],
     ):
         """Validates entities in terms of offset spans"""
-
-        def adjust_span_bounds(start, end):
-            if start < 0:
-                start = 0
-            if entity.end > len(self.text):
-                end = len(self.text)
-            while start <= len(self.text) and not self.text[start].strip():
-                start += 1
-            while not self.text[end - 1].strip():
-                end -= 1
-            return start, end
-
         if annotation:
             for entity in annotation.entities:
-                entity.start, entity.end = adjust_span_bounds(entity.start, entity.end)
                 mention = self.text[entity.start : entity.end]
                 assert len(mention) > 0, f"Empty offset defined for entity {entity}"
 

tests/server/token_classification/test_model.py

@@ -103,7 +103,7 @@ def test_model_dict():
         "raw_text": "This is  a  great  space",
         "status": "Default",
         "text": "This is  a  great  space",
-        "tokens": ["This", "is", " ", "a", " ", "great", " ", "space"],
+        "tokens": ["This", "is", "a", "great", "space"],
     }