From f7ab2029096c46f162df7077c2b4f3420d238ee2 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 17:17:26 +0200 Subject: [PATCH 01/23] Add support for CVAT annotation format --- src/labelformat/formats/cvat.py | 216 ++++++++++++++++++++++++++++++++ tests/unit/formats/test_cvat.py | 178 ++++++++++++++++++++++++++ 2 files changed, 394 insertions(+) create mode 100644 src/labelformat/formats/cvat.py create mode 100644 tests/unit/formats/test_cvat.py diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py new file mode 100644 index 0000000..4343153 --- /dev/null +++ b/src/labelformat/formats/cvat.py @@ -0,0 +1,216 @@ +import logging +import xml.etree.ElementTree as ET +from argparse import ArgumentParser +from collections.abc import Iterable, Sequence +from pathlib import Path + +from labelformat.cli.registry import Task, cli_register +from labelformat.model.bounding_box import BoundingBox, BoundingBoxFormat +from labelformat.model.category import Category +from labelformat.model.image import Image +from labelformat.model.object_detection import ( + ImageObjectDetection, + ObjectDetectionInput, + ObjectDetectionOutput, + SingleObjectDetection, +) +from labelformat.types import ParseError + +logger = logging.getLogger(__name__) + + +class _CVATBaseInput: + @staticmethod + def add_cli_arguments(parser: ArgumentParser) -> None: + parser.add_argument( + "--input-file", + type=Path, + required=True, + help="Path to input CVAT XML annotations file", + ) + + def __init__(self, input_file: Path) -> None: + try: + self._data = ET.parse(input_file).getroot() + except ET.ParseError as ex: + raise ParseError( + f"Could not parse XML file {input_file}: {str(ex)}" + ) from ex + self._categories = _get_categories(self._data) + + def get_categories(self) -> Iterable[Category]: + return self._categories + + def get_images(self) -> Iterable[Image]: + for label in self.get_labels(): + yield label.image + + +@cli_register(format="cvat", task=Task.OBJECT_DETECTION) +class CVATObjectDetectionInput(_CVATBaseInput, ObjectDetectionInput): + def get_labels(self) -> Iterable[ImageObjectDetection]: + xml_images = self._data.findall("image") + for xml_image in xml_images: + try: + image = _parse_image( + xml_root=xml_image, + ) + objects = _parse_object( + xml_root=xml_image, + categories=self._categories, + ) + except ParseError as ex: + raise ParseError(f"Could not parse XML file : {str(ex)}") from ex + + yield ImageObjectDetection( + image=image, + objects=objects, + ) + + +@cli_register(format="cvat", task=Task.OBJECT_DETECTION) +class CVATObjectDetectionOutput(ObjectDetectionOutput): + @staticmethod + def add_cli_arguments(parser: ArgumentParser) -> None: + parser.add_argument( + "--output-folder", + type=Path, + required=True, + help="Output folder to store generated CVAT XML annotations file", + ) + parser.add_argument( + "--output-annotation-scope ", + choices=["task", "job", "project"], + default="task", + help="Define the annotation scope to determine the XML structure: 'task', 'job', or 'project'.", + ) + + def __init__(self, output_folder: Path, annotation_scope: str) -> None: + self._output_folder = output_folder + self._annotation_scope = annotation_scope + + def save(self, label_input: ObjectDetectionInput) -> None: + # Write config file. + self._output_folder.mkdir(parents=True, exist_ok=True) + root = ET.Element("annotations") + + # Add meta information with labels + meta = ET.SubElement(root, "meta") + task = ET.SubElement(meta, self._annotation_scope) + labels = ET.SubElement(task, "labels") + + # Adding categories as labels + for category in label_input.get_categories(): + label = ET.SubElement(labels, "label") + name = ET.SubElement(label, "name") + name.text = category.name + + for label in label_input.get_labels(): + image_elem = ET.SubElement( + root, + "image", + { + "id": str(label.image.id), + "name": label.image.filename, + "width": str(label.image.width), + "height": str(label.image.height), + }, + ) + + for obj in label.objects: + bbox = obj.box + ET.SubElement( + image_elem, + "box", + { + "label": obj.category.name, + "xtl": str(bbox.xmin), + "ytl": str(bbox.ymin), + "xbr": str(bbox.xmax), + "ybr": str(bbox.ymax), + }, + ) + + tree = ET.ElementTree(root) + label_path = (self._output_folder / "annotations").with_suffix(".xml") + tree.write(label_path, encoding="utf-8", xml_declaration=True, short_empty_elements=False) + + +def _get_categories(xml_root: ET.Element) -> Sequence[Category]: + label_paths = ["meta/task/labels", "meta/job/labels", "meta/project/labels"] + for path in label_paths: + xml_labels = xml_root.find(path) + if xml_labels is not None: + xml_objects = xml_labels.findall("label") + categories = [ + Category( + id=index, name=_xml_text_or_raise(_xml_find_or_raise(label, "name")) + ) + for index, label in enumerate(xml_objects) + ] + return categories + raise ParseError( + f"Could not find labels at any of the provided paths: {', '.join(label_paths)}" + ) + + +def _parse_image(xml_root: ET.Element) -> Image: + _validate_required_attributes(xml_root, ["name", "id", "width", "height"]) + + return Image( + id=int(xml_root.get("id")), + filename=xml_root.get("name"), + width=int(xml_root.get("width")), + height=int(xml_root.get("height")), + ) + + +def _parse_object( + categories: Sequence[Category], xml_root: ET.Element +) -> Sequence[SingleObjectDetection]: + objects = [] + xml_boxes = xml_root.findall("box") + for xml_box in xml_boxes: + _validate_required_attributes(xml_box, ["label", "xtl", "ytl", "xbr", "ybr"]) + + label = xml_box.get("label") + category = next((cat for cat in categories if cat.name == label), None) + if category is None: + raise ParseError(f"Unknown category name '{label}'.") + bbox = [float(xml_box.get(attr)) for attr in ["xtl", "ytl", "xbr", "ybr"]] + + objects.append( + SingleObjectDetection( + category=category, + box=BoundingBox.from_format( + bbox=bbox, + format=BoundingBoxFormat.XYXY, + ), + ) + ) + + return objects + + +def _xml_find_or_raise(elem: ET.Element, path: str) -> ET.Element: + found_elem = elem.find(path=path) + if found_elem is None: + raise ParseError(f"Missing field '{path}' in XML.") + return found_elem + + +def _xml_text_or_raise(elem: ET.Element) -> str: + text = elem.text + if text is None: + raise ParseError( + f"Missing text content for XML element: {ET.tostring(elem, encoding='unicode')}" + ) + return text + + +def _validate_required_attributes( + xml_elem: ET.Element, required_attributes: Sequence[str] +) -> None: + missing_attrs = [attr for attr in required_attributes if xml_elem.get(attr) is None] + if missing_attrs: + raise ParseError(f"Missing required attributes: {', '.join(missing_attrs)}") diff --git a/tests/unit/formats/test_cvat.py b/tests/unit/formats/test_cvat.py new file mode 100644 index 0000000..985a017 --- /dev/null +++ b/tests/unit/formats/test_cvat.py @@ -0,0 +1,178 @@ +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +from labelformat.formats.cvat import CVATObjectDetectionInput, CVATObjectDetectionOutput +from labelformat.model.bounding_box import BoundingBox +from labelformat.model.category import Category +from labelformat.model.image import Image +from labelformat.model.object_detection import ( + ImageObjectDetection, + SingleObjectDetection, +) +from labelformat.types import ParseError + + +# Helper for creating temp XML files +def create_xml_file(tmp_path: Path, content: str) -> Path: + xml_path = tmp_path / "labels" / "annotations.xml" + xml_path.parent.mkdir(parents=True, exist_ok=True) + xml_path.write_text(content.strip()) + return xml_path + + +class TestCVATObjectDetectionInput: + @pytest.mark.parametrize("annotation_scope", ["task", "project", "job"]) + def test_get_labels(self, tmp_path: Path, annotation_scope: str) -> None: + annotation = f""" + + 1.1 + + <{annotation_scope}> + + + + + + + + + + + """ + + xml_path = create_xml_file(tmp_path, annotation) + label_input = CVATObjectDetectionInput(xml_path) + + # Validate categories + categories = label_input.get_categories() + assert categories == [ + Category(id=0, name="label1"), + Category(id=1, name="label2"), + ] + + # Validate labels + labels = list(label_input.get_labels()) + assert labels == [ + ImageObjectDetection( + image=Image(id=0, filename="img0.jpg", width=10, height=8), + objects=[ + SingleObjectDetection( + category=Category(id=0, name="label1"), + box=BoundingBox(xmin=4.0, ymin=0.0, xmax=4.0, ymax=2.0), + ) + ], + ) + ] + + def test_invalid_xml(self, tmp_path: Path) -> None: + invalid_annotation = """ + + + + + + + + + + + + + """ + xml_path = create_xml_file(tmp_path, invalid_annotation) + + with pytest.raises(ValueError, match="could not convert string to float"): + label_input = CVATObjectDetectionInput(xml_path) + list(label_input.get_labels()) + + def test_missing_attributes_for_image(self, tmp_path: Path) -> None: + invalid_annotation = """ + + + + + + + + + + + + + """ + xml_path = create_xml_file(tmp_path, invalid_annotation) + + with pytest.raises( + ParseError, + match="Could not parse XML file : Missing required attributes: height", + ): + label_input = CVATObjectDetectionInput(xml_path) + list(label_input.get_labels()) + + +class TestCVATObjectDetectionOutput: + @pytest.mark.parametrize("annotation_scope", ["task", "project", "job"]) + def test_cyclic_load_save(self, tmp_path: Path, annotation_scope: str) -> None: + annotation = f""" + + + <{annotation_scope}> + + + + + + + + + + + """ + + xml_path = create_xml_file(tmp_path, annotation) + label_input = CVATObjectDetectionInput(xml_path) + output_folder = tmp_path / "labels" + + CVATObjectDetectionOutput( + output_folder=output_folder, annotation_scope=annotation_scope + ).save(label_input=label_input) + + assert output_folder.exists() + assert output_folder.is_dir() + filepaths = list(output_folder.glob("**/*.xml")) + assert len(filepaths) == 1 + path = filepaths[0] + + assert path == tmp_path / "labels" / "annotations.xml" + + contents = path.read_text().replace(" ", "").replace("\n", "") + annotation = annotation.replace(" ", "").replace("\n", "") + assert contents == annotation + + def test_output_missing_labels(self, tmp_path: Path) -> None: + annotation = """ + + + + + + + + + + + """ + + xml_path = create_xml_file(tmp_path, annotation) + label_input = CVATObjectDetectionInput(xml_path) + output_folder = tmp_path / "labels" + + with pytest.raises( + ParseError, + match="Could not parse XML file : Unknown category name 'dummy'.", + ): + CVATObjectDetectionOutput( + output_folder=output_folder, annotation_scope="task" + ).save(label_input=label_input) From ed71ac3309c3fe57e1cb903312b9ecf96cd38bd4 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 17:21:35 +0200 Subject: [PATCH 02/23] format --- src/labelformat/formats/cvat.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 4343153..d2912f3 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -133,7 +133,12 @@ def save(self, label_input: ObjectDetectionInput) -> None: tree = ET.ElementTree(root) label_path = (self._output_folder / "annotations").with_suffix(".xml") - tree.write(label_path, encoding="utf-8", xml_declaration=True, short_empty_elements=False) + tree.write( + label_path, + encoding="utf-8", + xml_declaration=True, + short_empty_elements=False, + ) def _get_categories(xml_root: ET.Element) -> Sequence[Category]: From e120a76c296a25ea1ee150ff3b8a0ec340465363 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 17:31:52 +0200 Subject: [PATCH 03/23] update --- src/labelformat/formats/cvat.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index d2912f3..2330e2c 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -41,13 +41,15 @@ def __init__(self, input_file: Path) -> None: def get_categories(self) -> Iterable[Category]: return self._categories + + +@cli_register(format="cvat", task=Task.OBJECT_DETECTION) +class CVATObjectDetectionInput(_CVATBaseInput, ObjectDetectionInput): def get_images(self) -> Iterable[Image]: for label in self.get_labels(): yield label.image -@cli_register(format="cvat", task=Task.OBJECT_DETECTION) -class CVATObjectDetectionInput(_CVATBaseInput, ObjectDetectionInput): def get_labels(self) -> Iterable[ImageObjectDetection]: xml_images = self._data.findall("image") for xml_image in xml_images: From 87f0d12de25316f637b384c89eaeaf3c0813f8c0 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 17:33:32 +0200 Subject: [PATCH 04/23] format --- src/labelformat/formats/cvat.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 2330e2c..647967f 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -42,14 +42,12 @@ def get_categories(self) -> Iterable[Category]: return self._categories - @cli_register(format="cvat", task=Task.OBJECT_DETECTION) class CVATObjectDetectionInput(_CVATBaseInput, ObjectDetectionInput): def get_images(self) -> Iterable[Image]: for label in self.get_labels(): yield label.image - def get_labels(self) -> Iterable[ImageObjectDetection]: xml_images = self._data.findall("image") for xml_image in xml_images: From 7db1f8f008c5cc46e04e18053cd5c7a4bc295f87 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 17:40:29 +0200 Subject: [PATCH 05/23] remove sequence --- src/labelformat/formats/cvat.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 647967f..9c6b077 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -1,8 +1,8 @@ import logging import xml.etree.ElementTree as ET from argparse import ArgumentParser -from collections.abc import Iterable, Sequence from pathlib import Path +from typing import Iterable, List from labelformat.cli.registry import Task, cli_register from labelformat.model.bounding_box import BoundingBox, BoundingBoxFormat @@ -141,7 +141,7 @@ def save(self, label_input: ObjectDetectionInput) -> None: ) -def _get_categories(xml_root: ET.Element) -> Sequence[Category]: +def _get_categories(xml_root: ET.Element) -> List[Category]: label_paths = ["meta/task/labels", "meta/job/labels", "meta/project/labels"] for path in label_paths: xml_labels = xml_root.find(path) @@ -171,8 +171,8 @@ def _parse_image(xml_root: ET.Element) -> Image: def _parse_object( - categories: Sequence[Category], xml_root: ET.Element -) -> Sequence[SingleObjectDetection]: + categories: List[Category], xml_root: ET.Element +) -> List[SingleObjectDetection]: objects = [] xml_boxes = xml_root.findall("box") for xml_box in xml_boxes: @@ -214,7 +214,7 @@ def _xml_text_or_raise(elem: ET.Element) -> str: def _validate_required_attributes( - xml_elem: ET.Element, required_attributes: Sequence[str] + xml_elem: ET.Element, required_attributes: List[str] ) -> None: missing_attrs = [attr for attr in required_attributes if xml_elem.get(attr) is None] if missing_attrs: From 43a0ecf5926d8af1f9e35c0125bf3ac9f2a3cf21 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 17:57:37 +0200 Subject: [PATCH 06/23] fix type check error --- src/labelformat/formats/cvat.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 9c6b077..7fb30e2 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -105,19 +105,19 @@ def save(self, label_input: ObjectDetectionInput) -> None: name = ET.SubElement(label, "name") name.text = category.name - for label in label_input.get_labels(): + for label_object in label_input.get_labels(): image_elem = ET.SubElement( root, "image", { - "id": str(label.image.id), - "name": label.image.filename, - "width": str(label.image.width), - "height": str(label.image.height), + "id": str(label_object.image.id), + "name": label_object.image.filename, + "width": str(label_object.image.width), + "height": str(label_object.image.height), }, ) - for obj in label.objects: + for obj in label_object.objects: bbox = obj.box ET.SubElement( image_elem, @@ -163,10 +163,10 @@ def _parse_image(xml_root: ET.Element) -> Image: _validate_required_attributes(xml_root, ["name", "id", "width", "height"]) return Image( - id=int(xml_root.get("id")), - filename=xml_root.get("name"), - width=int(xml_root.get("width")), - height=int(xml_root.get("height")), + id=int(_xml_attribute_text_or_raise(xml_root, "id")), + filename=_xml_attribute_text_or_raise(xml_root, "name"), + width=int(_xml_attribute_text_or_raise(xml_root, "width")), + height=int(_xml_attribute_text_or_raise(xml_root, "height")), ) @@ -182,7 +182,10 @@ def _parse_object( category = next((cat for cat in categories if cat.name == label), None) if category is None: raise ParseError(f"Unknown category name '{label}'.") - bbox = [float(xml_box.get(attr)) for attr in ["xtl", "ytl", "xbr", "ybr"]] + bbox = [ + float(_xml_attribute_text_or_raise(xml_box, attr)) + for attr in ["xtl", "ytl", "xbr", "ybr"] + ] objects.append( SingleObjectDetection( @@ -213,6 +216,13 @@ def _xml_text_or_raise(elem: ET.Element) -> str: return text +def _xml_attribute_text_or_raise(elem: ET.Element, attribute_name: str) -> str: + attribute_text = elem.get(attribute_name) + if attribute_text is None: + raise ParseError(f"Bad value for attribute: {attribute_name}") + return attribute_text + + def _validate_required_attributes( xml_elem: ET.Element, required_attributes: List[str] ) -> None: From 66d86a7b7e967b16b0bd0d355f9a5dc40c3357be Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 18:09:15 +0200 Subject: [PATCH 07/23] keep order of attributes for python 3.7 --- src/labelformat/formats/cvat.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 7fb30e2..1282351 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -1,6 +1,7 @@ import logging import xml.etree.ElementTree as ET from argparse import ArgumentParser +from collections import OrderedDict from pathlib import Path from typing import Iterable, List @@ -109,12 +110,14 @@ def save(self, label_input: ObjectDetectionInput) -> None: image_elem = ET.SubElement( root, "image", - { - "id": str(label_object.image.id), - "name": label_object.image.filename, - "width": str(label_object.image.width), - "height": str(label_object.image.height), - }, + OrderedDict( + [ + ("id", str(label_object.image.id)), + ("name", label_object.image.filename), + ("width", str(label_object.image.width)), + ("height", str(label_object.image.height)), + ] + ), ) for obj in label_object.objects: From 63e78be45186b7db973c08ff125b65496ab595c0 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 21:13:14 +0200 Subject: [PATCH 08/23] update xml compare --- tests/unit/formats/test_cvat.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/unit/formats/test_cvat.py b/tests/unit/formats/test_cvat.py index 985a017..2565640 100644 --- a/tests/unit/formats/test_cvat.py +++ b/tests/unit/formats/test_cvat.py @@ -111,6 +111,21 @@ def test_missing_attributes_for_image(self, tmp_path: Path) -> None: label_input = CVATObjectDetectionInput(xml_path) list(label_input.get_labels()) +def _compare_xml_elements(elem1: ET.Element, elem2: ET.Element) -> bool: + """Recursively compare two XML elements for tag, attributes, and text.""" + if elem1.tag != elem2.tag or elem1.text != elem2.text: + return False + + if elem1.attrib != elem2.attrib: + return False + + children1 = list(elem1) + children2 = list(elem2) + + if len(children1) != len(children2): + return False + + return all(_compare_xml_elements(c1, c2) for c1, c2 in zip(children1, children2)) class TestCVATObjectDetectionOutput: @pytest.mark.parametrize("annotation_scope", ["task", "project", "job"]) @@ -147,9 +162,14 @@ def test_cyclic_load_save(self, tmp_path: Path, annotation_scope: str) -> None: assert path == tmp_path / "labels" / "annotations.xml" - contents = path.read_text().replace(" ", "").replace("\n", "") - annotation = annotation.replace(" ", "").replace("\n", "") - assert contents == annotation + annotation = annotation.replace("\n", "") + # Compare XML structure. + input_tree = ET.parse(xml_path)#ET.ElementTree(ET.fromstring(annotation)) + output_tree = ET.parse(path) + + assert _compare_xml_elements( + input_tree.getroot(), output_tree.getroot() + ), "The output XML structure doesn't match the input XML." def test_output_missing_labels(self, tmp_path: Path) -> None: annotation = """ From 63fbe4a00ac7f90a7e538ddc1ec050b7d9f4835a Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Thu, 6 Mar 2025 21:18:34 +0200 Subject: [PATCH 09/23] clean up --- tests/unit/formats/test_cvat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/formats/test_cvat.py b/tests/unit/formats/test_cvat.py index 2565640..475e7e4 100644 --- a/tests/unit/formats/test_cvat.py +++ b/tests/unit/formats/test_cvat.py @@ -111,6 +111,7 @@ def test_missing_attributes_for_image(self, tmp_path: Path) -> None: label_input = CVATObjectDetectionInput(xml_path) list(label_input.get_labels()) + def _compare_xml_elements(elem1: ET.Element, elem2: ET.Element) -> bool: """Recursively compare two XML elements for tag, attributes, and text.""" if elem1.tag != elem2.tag or elem1.text != elem2.text: @@ -127,6 +128,7 @@ def _compare_xml_elements(elem1: ET.Element, elem2: ET.Element) -> bool: return all(_compare_xml_elements(c1, c2) for c1, c2 in zip(children1, children2)) + class TestCVATObjectDetectionOutput: @pytest.mark.parametrize("annotation_scope", ["task", "project", "job"]) def test_cyclic_load_save(self, tmp_path: Path, annotation_scope: str) -> None: @@ -164,7 +166,7 @@ def test_cyclic_load_save(self, tmp_path: Path, annotation_scope: str) -> None: annotation = annotation.replace("\n", "") # Compare XML structure. - input_tree = ET.parse(xml_path)#ET.ElementTree(ET.fromstring(annotation)) + input_tree = ET.parse(xml_path) output_tree = ET.parse(path) assert _compare_xml_elements( From 7879a59e6cde70b495be84cf2789e68a9c05bfea Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Fri, 7 Mar 2025 09:25:39 +0200 Subject: [PATCH 10/23] no nned for OrderedDict anymore as the xml comparison changed --- src/labelformat/formats/cvat.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 1282351..7fb30e2 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -1,7 +1,6 @@ import logging import xml.etree.ElementTree as ET from argparse import ArgumentParser -from collections import OrderedDict from pathlib import Path from typing import Iterable, List @@ -110,14 +109,12 @@ def save(self, label_input: ObjectDetectionInput) -> None: image_elem = ET.SubElement( root, "image", - OrderedDict( - [ - ("id", str(label_object.image.id)), - ("name", label_object.image.filename), - ("width", str(label_object.image.width)), - ("height", str(label_object.image.height)), - ] - ), + { + "id": str(label_object.image.id), + "name": label_object.image.filename, + "width": str(label_object.image.width), + "height": str(label_object.image.height), + }, ) for obj in label_object.objects: From 2c496c345b55174c4d4f04f477069b54e166e395 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Fri, 7 Mar 2025 09:29:12 +0200 Subject: [PATCH 11/23] Update error message --- src/labelformat/formats/cvat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 7fb30e2..fc6c712 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -219,7 +219,7 @@ def _xml_text_or_raise(elem: ET.Element) -> str: def _xml_attribute_text_or_raise(elem: ET.Element, attribute_name: str) -> str: attribute_text = elem.get(attribute_name) if attribute_text is None: - raise ParseError(f"Bad value for attribute: {attribute_name}") + raise ParseError(f"Could not read attribute: '{attribute_name}'") return attribute_text From 8a2ead09fdd6b3f3b1db0c54e5ad8fb6bc8f446a Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 11 Mar 2025 10:31:36 +0200 Subject: [PATCH 12/23] use Pydantic XML --- src/labelformat/formats/cvat.py | 308 +++++++++++++++----------------- tests/unit/formats/test_cvat.py | 73 ++++---- 2 files changed, 181 insertions(+), 200 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index fc6c712..bbc261d 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -2,7 +2,9 @@ import xml.etree.ElementTree as ET from argparse import ArgumentParser from pathlib import Path -from typing import Iterable, List +from typing import Dict, Iterable, List, Optional + +from pydantic_xml import BaseXmlModel, attr, element from labelformat.cli.registry import Task, cli_register from labelformat.model.bounding_box import BoundingBox, BoundingBoxFormat @@ -18,6 +20,55 @@ logger = logging.getLogger(__name__) +# --- Pydantic XML models --- + + +class CVATLabel(BaseXmlModel, tag="label"): # type: ignore + name: str = element() + + +class CVATLabels(BaseXmlModel, tag="labels"): # type: ignore + label_list: List[CVATLabel] = element(tag="label") + + +class CVATTask(BaseXmlModel, tag="task"): # type: ignore + labels: Optional[CVATLabels] = element(tag="labels") + + +class CVATJob(BaseXmlModel, tag="job"): # type: ignore + labels: Optional[CVATLabels] = element(tag="labels") + + +class CVATProject(BaseXmlModel, tag="project"): # type: ignore + labels: Optional[CVATLabels] = element(tag="labels") + + +class CVATMeta(BaseXmlModel, tag="meta"): # type: ignore + task: Optional[CVATTask] = element(default=None) + job: Optional[CVATJob] = element(default=None) + project: Optional[CVATProject] = element(default=None) + + +class CVATBox(BaseXmlModel, tag="box"): # type: ignore + label: str = attr() + xtl: float = attr() + ytl: float = attr() + xbr: float = attr() + ybr: float = attr() + + +class CVATImage(BaseXmlModel, tag="image"): # type: ignore + id: int = attr() + name: str = attr() # Filename + width: int = attr() + height: int = attr() + boxes: list[CVATBox] = element(tag="box", default=[]) # type: ignore + + +class CVATAnnotations(BaseXmlModel, tag="annotations", search_mode="unordered"): # type: ignore + meta: CVATMeta = element() + images: list[CVATImage] = element(tag="image", default=[]) # type: ignore + class _CVATBaseInput: @staticmethod @@ -26,50 +77,73 @@ def add_cli_arguments(parser: ArgumentParser) -> None: "--input-file", type=Path, required=True, - help="Path to input CVAT XML annotations file", + help="Path to input CVAT XML file", ) def __init__(self, input_file: Path) -> None: try: - self._data = ET.parse(input_file).getroot() - except ET.ParseError as ex: - raise ParseError( + self._data = CVATAnnotations.from_xml(input_file.read_text()) + except Exception as ex: + raise ValueError( f"Could not parse XML file {input_file}: {str(ex)}" ) from ex - self._categories = _get_categories(self._data) - def get_categories(self) -> Iterable[Category]: - return self._categories + def get_categories(self) -> Iterable["Category"]: + meta = self._data.meta + labels: Optional[List[CVATLabel]] = None + if meta.task is not None and meta.task.labels: + labels = meta.task.labels.label_list + elif meta.job is not None and meta.job.labels: + labels = meta.job.labels.label_list + elif meta.project is not None and meta.project.labels: + labels = meta.project.labels.label_list + if labels is None: + raise ValueError( + "Could not find labels in meta/task, meta/job, or meta/project" + ) + for idx, label in enumerate(labels, start=1): + yield Category(id=idx, name=label.name) + + def get_images(self) -> Iterable["Image"]: + for img in self._data.images: + yield Image( + id=img.id, + filename=img.name, + width=img.width, + height=img.height, + ) @cli_register(format="cvat", task=Task.OBJECT_DETECTION) class CVATObjectDetectionInput(_CVATBaseInput, ObjectDetectionInput): - def get_images(self) -> Iterable[Image]: - for label in self.get_labels(): - yield label.image - - def get_labels(self) -> Iterable[ImageObjectDetection]: - xml_images = self._data.findall("image") - for xml_image in xml_images: - try: - image = _parse_image( - xml_root=xml_image, + def get_labels(self) -> Iterable["ImageObjectDetection"]: + category_by_name: Dict[str, Category] = { + cat.name: cat for cat in self.get_categories() + } + for img in self._data.images: + objects = [] + for box in img.boxes: + cat = category_by_name.get(box.label) + if cat is None: + raise ParseError(f"Unknown category name '{box.label}'.") + objects.append( + SingleObjectDetection( + category=cat, + box=BoundingBox.from_format( + bbox=[box.xtl, box.ytl, box.xbr, box.ybr], + format=BoundingBoxFormat.XYXY, + ), + ) ) - objects = _parse_object( - xml_root=xml_image, - categories=self._categories, - ) - except ParseError as ex: - raise ParseError(f"Could not parse XML file : {str(ex)}") from ex - yield ImageObjectDetection( - image=image, + image=Image( + id=img.id, filename=img.name, width=img.width, height=img.height + ), objects=objects, ) -@cli_register(format="cvat", task=Task.OBJECT_DETECTION) -class CVATObjectDetectionOutput(ObjectDetectionOutput): +class _CVATBaseOutput: @staticmethod def add_cli_arguments(parser: ArgumentParser) -> None: parser.add_argument( @@ -89,143 +163,51 @@ def __init__(self, output_folder: Path, annotation_scope: str) -> None: self._output_folder = output_folder self._annotation_scope = annotation_scope + +@cli_register(format="cvat", task=Task.OBJECT_DETECTION) +class CVATObjectDetectionOutput(_CVATBaseOutput, ObjectDetectionOutput): def save(self, label_input: ObjectDetectionInput) -> None: - # Write config file. - self._output_folder.mkdir(parents=True, exist_ok=True) - root = ET.Element("annotations") - - # Add meta information with labels - meta = ET.SubElement(root, "meta") - task = ET.SubElement(meta, self._annotation_scope) - labels = ET.SubElement(task, "labels") - - # Adding categories as labels - for category in label_input.get_categories(): - label = ET.SubElement(labels, "label") - name = ET.SubElement(label, "name") - name.text = category.name - - for label_object in label_input.get_labels(): - image_elem = ET.SubElement( - root, - "image", - { - "id": str(label_object.image.id), - "name": label_object.image.filename, - "width": str(label_object.image.width), - "height": str(label_object.image.height), - }, + images = [ + CVATImage( + id=label.image.id, + name=label.image.filename, + width=label.image.width, + height=label.image.height, + boxes=[ + CVATBox( + label=obj.category.name, + xtl=obj.box.xmin, + ytl=obj.box.ymin, + xbr=obj.box.xmax, + ybr=obj.box.ymax, + ) + for obj in label.objects + ], ) - - for obj in label_object.objects: - bbox = obj.box - ET.SubElement( - image_elem, - "box", - { - "label": obj.category.name, - "xtl": str(bbox.xmin), - "ytl": str(bbox.ymin), - "xbr": str(bbox.xmax), - "ybr": str(bbox.ymax), - }, - ) - - tree = ET.ElementTree(root) - label_path = (self._output_folder / "annotations").with_suffix(".xml") - tree.write( - label_path, - encoding="utf-8", - xml_declaration=True, - short_empty_elements=False, - ) - - -def _get_categories(xml_root: ET.Element) -> List[Category]: - label_paths = ["meta/task/labels", "meta/job/labels", "meta/project/labels"] - for path in label_paths: - xml_labels = xml_root.find(path) - if xml_labels is not None: - xml_objects = xml_labels.findall("label") - categories = [ - Category( - id=index, name=_xml_text_or_raise(_xml_find_or_raise(label, "name")) - ) - for index, label in enumerate(xml_objects) - ] - return categories - raise ParseError( - f"Could not find labels at any of the provided paths: {', '.join(label_paths)}" - ) - - -def _parse_image(xml_root: ET.Element) -> Image: - _validate_required_attributes(xml_root, ["name", "id", "width", "height"]) - - return Image( - id=int(_xml_attribute_text_or_raise(xml_root, "id")), - filename=_xml_attribute_text_or_raise(xml_root, "name"), - width=int(_xml_attribute_text_or_raise(xml_root, "width")), - height=int(_xml_attribute_text_or_raise(xml_root, "height")), - ) - - -def _parse_object( - categories: List[Category], xml_root: ET.Element -) -> List[SingleObjectDetection]: - objects = [] - xml_boxes = xml_root.findall("box") - for xml_box in xml_boxes: - _validate_required_attributes(xml_box, ["label", "xtl", "ytl", "xbr", "ybr"]) - - label = xml_box.get("label") - category = next((cat for cat in categories if cat.name == label), None) - if category is None: - raise ParseError(f"Unknown category name '{label}'.") - bbox = [ - float(_xml_attribute_text_or_raise(xml_box, attr)) - for attr in ["xtl", "ytl", "xbr", "ybr"] + for label in label_input.get_labels() ] - - objects.append( - SingleObjectDetection( - category=category, - box=BoundingBox.from_format( - bbox=bbox, - format=BoundingBoxFormat.XYXY, - ), - ) - ) - - return objects - - -def _xml_find_or_raise(elem: ET.Element, path: str) -> ET.Element: - found_elem = elem.find(path=path) - if found_elem is None: - raise ParseError(f"Missing field '{path}' in XML.") - return found_elem - - -def _xml_text_or_raise(elem: ET.Element) -> str: - text = elem.text - if text is None: - raise ParseError( - f"Missing text content for XML element: {ET.tostring(elem, encoding='unicode')}" + actual_labels = list(label_input.get_categories()) + labels = CVATLabels( + label_list=[CVATLabel(name=cat.name) for cat in actual_labels] ) - return text - + if self._annotation_scope == "task": + meta = CVATMeta(task=CVATTask(labels=labels)) + elif self._annotation_scope == "project": + meta = CVATMeta(project=CVATProject(labels=labels)) + elif self._annotation_scope == "job": + meta = CVATMeta(job=CVATJob(labels=labels)) + else: + raise ValueError(f"Unknown annotation_scope: {self._annotation_scope}") + annotations = CVATAnnotations(meta=meta, images=images) -def _xml_attribute_text_or_raise(elem: ET.Element, attribute_name: str) -> str: - attribute_text = elem.get(attribute_name) - if attribute_text is None: - raise ParseError(f"Could not read attribute: '{attribute_name}'") - return attribute_text - - -def _validate_required_attributes( - xml_elem: ET.Element, required_attributes: List[str] -) -> None: - missing_attrs = [attr for attr in required_attributes if xml_elem.get(attr) is None] - if missing_attrs: - raise ParseError(f"Missing required attributes: {', '.join(missing_attrs)}") + self._output_folder.mkdir(parents=True, exist_ok=True) + output_file = self._output_folder / "annotations.xml" + # Convert the pydantic model to XML string + xml_bytes = annotations.to_xml() + if isinstance(xml_bytes, bytes): # Ensure it's bytes before decoding + xml_string = xml_bytes.decode("utf-8") + else: + xml_string = xml_bytes # In case it's already a string + # Save it as a string (ensure you're writing the XML as a string) + with output_file.open("w", encoding="utf-8") as f: + f.write(xml_string) diff --git a/tests/unit/formats/test_cvat.py b/tests/unit/formats/test_cvat.py index 475e7e4..430658a 100644 --- a/tests/unit/formats/test_cvat.py +++ b/tests/unit/formats/test_cvat.py @@ -28,17 +28,18 @@ def test_get_labels(self, tmp_path: Path, annotation_scope: str) -> None: annotation = f""" 1.1 - + <{annotation_scope}> - - + + - + + 1.1 """ @@ -46,10 +47,10 @@ def test_get_labels(self, tmp_path: Path, annotation_scope: str) -> None: label_input = CVATObjectDetectionInput(xml_path) # Validate categories - categories = label_input.get_categories() + categories = list(label_input.get_categories()) assert categories == [ - Category(id=0, name="label1"), - Category(id=1, name="label2"), + Category(id=1, name="label1"), + Category(id=2, name="label2"), ] # Validate labels @@ -59,7 +60,7 @@ def test_get_labels(self, tmp_path: Path, annotation_scope: str) -> None: image=Image(id=0, filename="img0.jpg", width=10, height=8), objects=[ SingleObjectDetection( - category=Category(id=0, name="label1"), + category=Category(id=1, name="label1"), box=BoundingBox(xmin=4.0, ymin=0.0, xmax=4.0, ymax=2.0), ) ], @@ -83,7 +84,10 @@ def test_invalid_xml(self, tmp_path: Path) -> None: """ xml_path = create_xml_file(tmp_path, invalid_annotation) - with pytest.raises(ValueError, match="could not convert string to float"): + with pytest.raises( + ValueError, + match="Input should be a valid number, unable to parse string as a number", + ): label_input = CVATObjectDetectionInput(xml_path) list(label_input.get_labels()) @@ -105,12 +109,33 @@ def test_missing_attributes_for_image(self, tmp_path: Path) -> None: xml_path = create_xml_file(tmp_path, invalid_annotation) with pytest.raises( - ParseError, - match="Could not parse XML file : Missing required attributes: height", + ValueError, + match="validation error for CVATAnnotations\nimages.0.height", ): label_input = CVATObjectDetectionInput(xml_path) list(label_input.get_labels()) + def test_invalid_label(self, tmp_path: Path) -> None: + invalid_annotation = """ + + + + + + + + + + + + + """ + xml_path = create_xml_file(tmp_path, invalid_annotation) + + with pytest.raises(ParseError, match="Unknown category name 'label2'"): + label_input = CVATObjectDetectionInput(xml_path) + list(label_input.get_labels()) + def _compare_xml_elements(elem1: ET.Element, elem2: ET.Element) -> bool: """Recursively compare two XML elements for tag, attributes, and text.""" @@ -172,29 +197,3 @@ def test_cyclic_load_save(self, tmp_path: Path, annotation_scope: str) -> None: assert _compare_xml_elements( input_tree.getroot(), output_tree.getroot() ), "The output XML structure doesn't match the input XML." - - def test_output_missing_labels(self, tmp_path: Path) -> None: - annotation = """ - - - - - - - - - - - """ - - xml_path = create_xml_file(tmp_path, annotation) - label_input = CVATObjectDetectionInput(xml_path) - output_folder = tmp_path / "labels" - - with pytest.raises( - ParseError, - match="Could not parse XML file : Unknown category name 'dummy'.", - ): - CVATObjectDetectionOutput( - output_folder=output_folder, annotation_scope="task" - ).save(label_input=label_input) From c9d73e30d6d3b207bc9e6e98f128d044ac2b9733 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 11 Mar 2025 10:44:11 +0200 Subject: [PATCH 13/23] add pydantic-xml to project.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5327879..685c19e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ python = ">=3.8" tqdm = "*" pyyaml = "*" pillow = "*" +pydantic-xml = "*" [tool.poetry.group.dev.dependencies] mypy = "*" From 1bd898476fa46d197ba8f494a1b4617cabe40b1e Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 11 Mar 2025 10:46:20 +0200 Subject: [PATCH 14/23] update poetry.lock file --- poetry.lock | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 0b9afae..f9a134d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,19 @@ # This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.9\""} + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -927,6 +941,157 @@ files = [ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +[[package]] +name = "pydantic" +version = "2.10.6" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.10.6-py3-none-any.whl", hash = "sha256:427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584"}, + {file = "pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.27.2" +typing-extensions = ">=4.12.2" + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] + +[[package]] +name = "pydantic-core" +version = "2.27.2" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"}, + {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af"}, + {file = "pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4"}, + {file = "pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31"}, + {file = "pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc"}, + {file = "pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0"}, + {file = "pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b"}, + {file = "pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b"}, + {file = "pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b"}, + {file = "pydantic_core-2.27.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d3e8d504bdd3f10835468f29008d72fc8359d95c9c415ce6e767203db6127506"}, + {file = "pydantic_core-2.27.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:521eb9b7f036c9b6187f0b47318ab0d7ca14bd87f776240b90b21c1f4f149320"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85210c4d99a0114f5a9481b44560d7d1e35e32cc5634c656bc48e590b669b145"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d716e2e30c6f140d7560ef1538953a5cd1a87264c737643d481f2779fc247fe1"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f66d89ba397d92f840f8654756196d93804278457b5fbede59598a1f9f90b228"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:669e193c1c576a58f132e3158f9dfa9662969edb1a250c54d8fa52590045f046"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdbe7629b996647b99c01b37f11170a57ae675375b14b8c13b8518b8320ced5"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d262606bf386a5ba0b0af3b97f37c83d7011439e3dc1a9298f21efb292e42f1a"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cabb9bcb7e0d97f74df8646f34fc76fbf793b7f6dc2438517d7a9e50eee4f14d"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d2d63f1215638d28221f664596b1ccb3944f6e25dd18cd3b86b0a4c408d5ebb9"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bca101c00bff0adb45a833f8451b9105d9df18accb8743b08107d7ada14bd7da"}, + {file = "pydantic_core-2.27.2-cp38-cp38-win32.whl", hash = "sha256:f6f8e111843bbb0dee4cb6594cdc73e79b3329b526037ec242a3e49012495b3b"}, + {file = "pydantic_core-2.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:fd1aea04935a508f62e0d0ef1f5ae968774a32afc306fb8545e06f5ff5cdf3ad"}, + {file = "pydantic_core-2.27.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c10eb4f1659290b523af58fa7cffb452a61ad6ae5613404519aee4bfbf1df993"}, + {file = "pydantic_core-2.27.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef592d4bad47296fb11f96cd7dc898b92e795032b4894dfb4076cfccd43a9308"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61709a844acc6bf0b7dce7daae75195a10aac96a596ea1b776996414791ede4"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c5f762659e47fdb7b16956c71598292f60a03aa92f8b6351504359dbdba6cf"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c9775e339e42e79ec99c441d9730fccf07414af63eac2f0e48e08fd38a64d76"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57762139821c31847cfb2df63c12f725788bd9f04bc2fb392790959b8f70f118"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d1e85068e818c73e048fe28cfc769040bb1f475524f4745a5dc621f75ac7630"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:097830ed52fd9e427942ff3b9bc17fab52913b2f50f2880dc4a5611446606a54"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044a50963a614ecfae59bb1eaf7ea7efc4bc62f49ed594e18fa1e5d953c40e9f"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:4e0b4220ba5b40d727c7f879eac379b822eee5d8fff418e9d3381ee45b3b0362"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4f4bb20d75e9325cc9696c6802657b58bc1dbbe3022f32cc2b2b632c3fbb96"}, + {file = "pydantic_core-2.27.2-cp39-cp39-win32.whl", hash = "sha256:cca63613e90d001b9f2f9a9ceb276c308bfa2a43fafb75c8031c4f66039e8c6e"}, + {file = "pydantic_core-2.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:77d1bca19b0f7021b3a982e6f903dcd5b2b06076def36a652e3907f596e29f67"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c33939a82924da9ed65dab5a65d427205a73181d8098e79b6b426bdf8ad4e656"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00bad2484fa6bda1e216e7345a798bd37c68fb2d97558edd584942aa41b7d278"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817e2b40aba42bac6f457498dacabc568c3b7a986fc9ba7c8d9d260b71485fb"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:251136cdad0cb722e93732cb45ca5299fb56e1344a833640bf93b2803f8d1bfd"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2088237af596f0a524d3afc39ab3b036e8adb054ee57cbb1dcf8e09da5b29cc"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d4041c0b966a84b4ae7a09832eb691a35aec90910cd2dbe7a208de59be77965b"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8083d4e875ebe0b864ffef72a4304827015cff328a1be6e22cc850753bfb122b"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f141ee28a0ad2123b6611b6ceff018039df17f32ada8b534e6aa039545a3efb2"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7d0c8399fcc1848491f00e0314bd59fb34a9c008761bcb422a057670c3f65e35"}, + {file = "pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pydantic-xml" +version = "2.14.2" +description = "pydantic xml extension" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_xml-2.14.2-py3-none-any.whl", hash = "sha256:c21f5b777ae39d6cb6da7474b3f97a90d42a22cdc8dc3db7cf53d9b1ba119a33"}, + {file = "pydantic_xml-2.14.2.tar.gz", hash = "sha256:73206dfd623e838791a612ef398834732bfa2b4b4b853b0126d1298f71199d78"}, +] + +[package.dependencies] +pydantic = ">=2.6.0,<2.10.0b1 || >2.10.0b1" +pydantic-core = ">=2.15.0" + +[package.extras] +docs = ["Sphinx (>=5.3.0,<6.0.0)", "furo (>=2022.12.7,<2023.0.0)", "sphinx-copybutton (>=0.5.1,<0.6.0)", "sphinx_design (>=0.3.0,<0.4.0)", "toml (>=0.10.2,<0.11.0)"] +lxml = ["lxml (>=4.9.0)"] + [[package]] name = "pyflakes" version = "2.5.0" @@ -1337,4 +1502,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.8" -content-hash = "cce97a3eba063cec408c554481efa57bff23a20a1baa86bd40590f9bffdc30e3" +content-hash = "9c717c3cef42122fb097449fa23aa714e345666637ae525a16d4ab027b21156d" From 18a4feaa211fbb73b445210e9dce6c9aaabe9757 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 11 Mar 2025 10:59:00 +0200 Subject: [PATCH 15/23] update --- src/labelformat/formats/cvat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index bbc261d..9fb4027 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -62,12 +62,12 @@ class CVATImage(BaseXmlModel, tag="image"): # type: ignore name: str = attr() # Filename width: int = attr() height: int = attr() - boxes: list[CVATBox] = element(tag="box", default=[]) # type: ignore + boxes: List[CVATBox] = element(tag="box", default=[]) # type: ignore class CVATAnnotations(BaseXmlModel, tag="annotations", search_mode="unordered"): # type: ignore meta: CVATMeta = element() - images: list[CVATImage] = element(tag="image", default=[]) # type: ignore + images: List[CVATImage] = element(tag="image", default=[]) # type: ignore class _CVATBaseInput: From 95f6eb2f05943c56a5ea5bc336a1cfc259dc0064 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 11 Mar 2025 11:02:03 +0200 Subject: [PATCH 16/23] remove unused "type: ignore" comment --- src/labelformat/formats/cvat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 9fb4027..d4b3c5e 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -62,12 +62,12 @@ class CVATImage(BaseXmlModel, tag="image"): # type: ignore name: str = attr() # Filename width: int = attr() height: int = attr() - boxes: List[CVATBox] = element(tag="box", default=[]) # type: ignore + boxes: List[CVATBox] = element(tag="box", default=[]) class CVATAnnotations(BaseXmlModel, tag="annotations", search_mode="unordered"): # type: ignore meta: CVATMeta = element() - images: List[CVATImage] = element(tag="image", default=[]) # type: ignore + images: List[CVATImage] = element(tag="image", default=[]) class _CVATBaseInput: From 206771d5a80636753daa9131f0c15d56a40e34f1 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 11 Mar 2025 13:37:33 +0200 Subject: [PATCH 17/23] update --- src/labelformat/formats/cvat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index d4b3c5e..13e8bb9 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -20,9 +20,8 @@ logger = logging.getLogger(__name__) -# --- Pydantic XML models --- - +# --- Pydantic XML models --- class CVATLabel(BaseXmlModel, tag="label"): # type: ignore name: str = element() @@ -186,9 +185,10 @@ def save(self, label_input: ObjectDetectionInput) -> None: ) for label in label_input.get_labels() ] - actual_labels = list(label_input.get_categories()) labels = CVATLabels( - label_list=[CVATLabel(name=cat.name) for cat in actual_labels] + label_list=[ + CVATLabel(name=cat.name) for cat in label_input.get_categories() + ] ) if self._annotation_scope == "task": meta = CVATMeta(task=CVATTask(labels=labels)) From d4803c87ea1fee40f60d75581c6d1f9e2695bcd6 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 11 Mar 2025 18:21:57 +0200 Subject: [PATCH 18/23] update after review --- src/labelformat/formats/cvat.py | 53 ++++++++++++++++++++----------- src/labelformat/types.py | 4 +++ tests/unit/formats/test_cvat.py | 55 +++++++++++++++++++++++---------- 3 files changed, 77 insertions(+), 35 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 13e8bb9..5d52c04 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -1,6 +1,7 @@ import logging import xml.etree.ElementTree as ET from argparse import ArgumentParser +from enum import Enum from pathlib import Path from typing import Dict, Iterable, List, Optional @@ -16,12 +17,12 @@ ObjectDetectionOutput, SingleObjectDetection, ) -from labelformat.types import ParseError +from labelformat.types import ArgumentError, ParseError logger = logging.getLogger(__name__) -# --- Pydantic XML models --- +# The following Pydantic XML models describe the structure of CVAT XML files. class CVATLabel(BaseXmlModel, tag="label"): # type: ignore name: str = element() @@ -80,12 +81,11 @@ def add_cli_arguments(parser: ArgumentParser) -> None: ) def __init__(self, input_file: Path) -> None: + xml_text = input_file.read_text() try: - self._data = CVATAnnotations.from_xml(input_file.read_text()) + self._data = CVATAnnotations.from_xml(xml_text) except Exception as ex: - raise ValueError( - f"Could not parse XML file {input_file}: {str(ex)}" - ) from ex + raise ValueError(f"Could not parse XML file {input_file}: {ex}") from ex def get_categories(self) -> Iterable["Category"]: meta = self._data.meta @@ -142,6 +142,16 @@ def get_labels(self) -> Iterable["ImageObjectDetection"]: ) +class AnnotationScope(Enum): + TASK = "task" + JOB = "job" + PROJECT = "project" + + @staticmethod + def allowed_values() -> str: + return ", ".join(scope.value for scope in AnnotationScope) + + class _CVATBaseOutput: @staticmethod def add_cli_arguments(parser: ArgumentParser) -> None: @@ -153,12 +163,17 @@ def add_cli_arguments(parser: ArgumentParser) -> None: ) parser.add_argument( "--output-annotation-scope ", - choices=["task", "job", "project"], - default="task", - help="Define the annotation scope to determine the XML structure: 'task', 'job', or 'project'.", + choices=[scope.value for scope in AnnotationScope], + default=AnnotationScope.TASK.value, + help="Define the annotation scope to determine the XML structure. Allowed values: " + + AnnotationScope.allowed_values(), ) - def __init__(self, output_folder: Path, annotation_scope: str) -> None: + def __init__(self, output_folder: Path, annotation_scope: AnnotationScope) -> None: + if not isinstance(annotation_scope, AnnotationScope): + raise ArgumentError( + f"annotation_scope must be one of the allowed values: {AnnotationScope.allowed_values()}" + ) self._output_folder = output_folder self._annotation_scope = annotation_scope @@ -190,24 +205,26 @@ def save(self, label_input: ObjectDetectionInput) -> None: CVATLabel(name=cat.name) for cat in label_input.get_categories() ] ) - if self._annotation_scope == "task": + if self._annotation_scope == AnnotationScope.TASK: meta = CVATMeta(task=CVATTask(labels=labels)) - elif self._annotation_scope == "project": + elif self._annotation_scope == AnnotationScope.PROJECT: meta = CVATMeta(project=CVATProject(labels=labels)) - elif self._annotation_scope == "job": + elif self._annotation_scope == AnnotationScope.JOB: meta = CVATMeta(job=CVATJob(labels=labels)) else: - raise ValueError(f"Unknown annotation_scope: {self._annotation_scope}") + raise ValueError( + f"Unknown annotation_scope: {self._annotation_scope}. Allowed values: {AnnotationScope.allowed_values()}." + ) annotations = CVATAnnotations(meta=meta, images=images) self._output_folder.mkdir(parents=True, exist_ok=True) output_file = self._output_folder / "annotations.xml" - # Convert the pydantic model to XML string + # Convert the Pydantic model to an XML format (as bytes or string). xml_bytes = annotations.to_xml() - if isinstance(xml_bytes, bytes): # Ensure it's bytes before decoding + # Ensure the XML output is a string — decode bytes or use as-is if it's already a string. + if isinstance(xml_bytes, bytes): xml_string = xml_bytes.decode("utf-8") else: - xml_string = xml_bytes # In case it's already a string - # Save it as a string (ensure you're writing the XML as a string) + xml_string = xml_bytes with output_file.open("w", encoding="utf-8") as f: f.write(xml_string) diff --git a/src/labelformat/types.py b/src/labelformat/types.py index 33d0d12..69cc483 100644 --- a/src/labelformat/types.py +++ b/src/labelformat/types.py @@ -5,3 +5,7 @@ class ParseError(Exception): pass + + +class ArgumentError(Exception): + pass diff --git a/tests/unit/formats/test_cvat.py b/tests/unit/formats/test_cvat.py index 430658a..0c5c149 100644 --- a/tests/unit/formats/test_cvat.py +++ b/tests/unit/formats/test_cvat.py @@ -3,7 +3,11 @@ import pytest -from labelformat.formats.cvat import CVATObjectDetectionInput, CVATObjectDetectionOutput +from labelformat.formats.cvat import ( + AnnotationScope, + CVATObjectDetectionInput, + CVATObjectDetectionOutput, +) from labelformat.model.bounding_box import BoundingBox from labelformat.model.category import Category from labelformat.model.image import Image @@ -11,7 +15,7 @@ ImageObjectDetection, SingleObjectDetection, ) -from labelformat.types import ParseError +from labelformat.types import ArgumentError, ParseError # Helper for creating temp XML files @@ -23,18 +27,23 @@ def create_xml_file(tmp_path: Path, content: str) -> Path: class TestCVATObjectDetectionInput: - @pytest.mark.parametrize("annotation_scope", ["task", "project", "job"]) - def test_get_labels(self, tmp_path: Path, annotation_scope: str) -> None: + @pytest.mark.parametrize( + "annotation_scope", + [AnnotationScope.TASK, AnnotationScope.PROJECT, AnnotationScope.JOB], + ) + def test_get_labels( + self, tmp_path: Path, annotation_scope: AnnotationScope + ) -> None: annotation = f""" 1.1 - <{annotation_scope}> + <{annotation_scope.value}> - + @@ -46,14 +55,14 @@ def test_get_labels(self, tmp_path: Path, annotation_scope: str) -> None: xml_path = create_xml_file(tmp_path, annotation) label_input = CVATObjectDetectionInput(xml_path) - # Validate categories + # Validate categories. categories = list(label_input.get_categories()) assert categories == [ Category(id=1, name="label1"), Category(id=2, name="label2"), ] - # Validate labels + # Validate labels. labels = list(label_input.get_labels()) assert labels == [ ImageObjectDetection( @@ -67,7 +76,7 @@ def test_get_labels(self, tmp_path: Path, annotation_scope: str) -> None: ) ] - def test_invalid_xml(self, tmp_path: Path) -> None: + def test___init___invalid_xml(self, tmp_path: Path) -> None: invalid_annotation = """ @@ -89,9 +98,8 @@ def test_invalid_xml(self, tmp_path: Path) -> None: match="Input should be a valid number, unable to parse string as a number", ): label_input = CVATObjectDetectionInput(xml_path) - list(label_input.get_labels()) - def test_missing_attributes_for_image(self, tmp_path: Path) -> None: + def test___init____missing_attributes_for_image(self, tmp_path: Path) -> None: invalid_annotation = """ @@ -113,9 +121,8 @@ def test_missing_attributes_for_image(self, tmp_path: Path) -> None: match="validation error for CVATAnnotations\nimages.0.height", ): label_input = CVATObjectDetectionInput(xml_path) - list(label_input.get_labels()) - def test_invalid_label(self, tmp_path: Path) -> None: + def test_get_labels_invalid_category_name(self, tmp_path: Path) -> None: invalid_annotation = """ @@ -155,17 +162,22 @@ def _compare_xml_elements(elem1: ET.Element, elem2: ET.Element) -> bool: class TestCVATObjectDetectionOutput: - @pytest.mark.parametrize("annotation_scope", ["task", "project", "job"]) - def test_cyclic_load_save(self, tmp_path: Path, annotation_scope: str) -> None: + @pytest.mark.parametrize( + "annotation_scope", + [AnnotationScope.TASK, AnnotationScope.PROJECT, AnnotationScope.JOB], + ) + def test_save_cyclic_load_and_save( + self, tmp_path: Path, annotation_scope: AnnotationScope + ) -> None: annotation = f""" - <{annotation_scope}> + <{annotation_scope.value}> - + @@ -197,3 +209,12 @@ def test_cyclic_load_save(self, tmp_path: Path, annotation_scope: str) -> None: assert _compare_xml_elements( input_tree.getroot(), output_tree.getroot() ), "The output XML structure doesn't match the input XML." + + def test__init__invalid_annotation_scope(self, tmp_path: Path) -> None: + with pytest.raises( + ArgumentError, + match="annotation_scope must be one of the allowed values: task, job, project", + ): + CVATObjectDetectionOutput( + output_folder=tmp_path, annotation_scope="invalid" + ) From ec9740c3069c86948a06ac49c4ded2f1d8107065 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Tue, 11 Mar 2025 18:25:10 +0200 Subject: [PATCH 19/23] remove test as it will produce mypy error --- tests/unit/formats/test_cvat.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/unit/formats/test_cvat.py b/tests/unit/formats/test_cvat.py index 0c5c149..9826d17 100644 --- a/tests/unit/formats/test_cvat.py +++ b/tests/unit/formats/test_cvat.py @@ -209,12 +209,3 @@ def test_save_cyclic_load_and_save( assert _compare_xml_elements( input_tree.getroot(), output_tree.getroot() ), "The output XML structure doesn't match the input XML." - - def test__init__invalid_annotation_scope(self, tmp_path: Path) -> None: - with pytest.raises( - ArgumentError, - match="annotation_scope must be one of the allowed values: task, job, project", - ): - CVATObjectDetectionOutput( - output_folder=tmp_path, annotation_scope="invalid" - ) From 14c81d72f8d01e91658e82c3d5307566be9618ae Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 12 Mar 2025 14:44:38 +0200 Subject: [PATCH 20/23] update type of output_annotation_scope parameter --- src/labelformat/formats/__init__.py | 3 +++ src/labelformat/formats/cvat.py | 38 ++++++++++++++++------------- src/labelformat/types.py | 4 --- tests/unit/formats/test_cvat.py | 4 +-- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/labelformat/formats/__init__.py b/src/labelformat/formats/__init__.py index 3d6f840..38a29dc 100644 --- a/src/labelformat/formats/__init__.py +++ b/src/labelformat/formats/__init__.py @@ -4,6 +4,7 @@ COCOObjectDetectionInput, COCOObjectDetectionOutput, ) +from labelformat.formats.cvat import CVATObjectDetectionInput, CVATObjectDetectionOutput from labelformat.formats.kitti import ( KittiObjectDetectionInput, KittiObjectDetectionOutput, @@ -53,6 +54,8 @@ "COCOInstanceSegmentationOutput", "COCOObjectDetectionInput", "COCOObjectDetectionOutput", + "CVATObjectDetectionInput", + "CVATObjectDetectionOutput", "KittiObjectDetectionInput", "KittiObjectDetectionOutput", "LabelboxObjectDetectionInput", diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 5d52c04..b06212f 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -1,9 +1,8 @@ import logging -import xml.etree.ElementTree as ET from argparse import ArgumentParser from enum import Enum from pathlib import Path -from typing import Dict, Iterable, List, Optional +from typing import Dict, Iterable, List, Literal, Optional from pydantic_xml import BaseXmlModel, attr, element @@ -17,39 +16,39 @@ ObjectDetectionOutput, SingleObjectDetection, ) -from labelformat.types import ArgumentError, ParseError +from labelformat.types import ParseError logger = logging.getLogger(__name__) # The following Pydantic XML models describe the structure of CVAT XML files. -class CVATLabel(BaseXmlModel, tag="label"): # type: ignore +class CVATLabel(BaseXmlModel, tag="label", search_mode="unordered"): # type: ignore name: str = element() -class CVATLabels(BaseXmlModel, tag="labels"): # type: ignore +class CVATLabels(BaseXmlModel, tag="labels", search_mode="unordered"): # type: ignore label_list: List[CVATLabel] = element(tag="label") -class CVATTask(BaseXmlModel, tag="task"): # type: ignore +class CVATTask(BaseXmlModel, tag="task", search_mode="unordered"): # type: ignore labels: Optional[CVATLabels] = element(tag="labels") -class CVATJob(BaseXmlModel, tag="job"): # type: ignore +class CVATJob(BaseXmlModel, tag="job", search_mode="unordered"): # type: ignore labels: Optional[CVATLabels] = element(tag="labels") -class CVATProject(BaseXmlModel, tag="project"): # type: ignore +class CVATProject(BaseXmlModel, tag="project", search_mode="unordered"): # type: ignore labels: Optional[CVATLabels] = element(tag="labels") -class CVATMeta(BaseXmlModel, tag="meta"): # type: ignore +class CVATMeta(BaseXmlModel, tag="meta", search_mode="unordered"): # type: ignore task: Optional[CVATTask] = element(default=None) job: Optional[CVATJob] = element(default=None) project: Optional[CVATProject] = element(default=None) -class CVATBox(BaseXmlModel, tag="box"): # type: ignore +class CVATBox(BaseXmlModel, tag="box", search_mode="unordered"): # type: ignore label: str = attr() xtl: float = attr() ytl: float = attr() @@ -57,7 +56,7 @@ class CVATBox(BaseXmlModel, tag="box"): # type: ignore ybr: float = attr() -class CVATImage(BaseXmlModel, tag="image"): # type: ignore +class CVATImage(BaseXmlModel, tag="image", search_mode="unordered"): # type: ignore id: int = attr() name: str = attr() # Filename width: int = attr() @@ -162,20 +161,25 @@ def add_cli_arguments(parser: ArgumentParser) -> None: help="Output folder to store generated CVAT XML annotations file", ) parser.add_argument( - "--output-annotation-scope ", + "--output-annotation-scope", choices=[scope.value for scope in AnnotationScope], - default=AnnotationScope.TASK.value, + default="task", help="Define the annotation scope to determine the XML structure. Allowed values: " + AnnotationScope.allowed_values(), ) - def __init__(self, output_folder: Path, annotation_scope: AnnotationScope) -> None: - if not isinstance(annotation_scope, AnnotationScope): - raise ArgumentError( + def __init__( + self, + output_folder: Path, + output_annotation_scope: Literal["task", "job", "project"], + ) -> None: + try: + self._annotation_scope = AnnotationScope(output_annotation_scope) + except ValueError: + raise ValueError( f"annotation_scope must be one of the allowed values: {AnnotationScope.allowed_values()}" ) self._output_folder = output_folder - self._annotation_scope = annotation_scope @cli_register(format="cvat", task=Task.OBJECT_DETECTION) diff --git a/src/labelformat/types.py b/src/labelformat/types.py index 69cc483..33d0d12 100644 --- a/src/labelformat/types.py +++ b/src/labelformat/types.py @@ -5,7 +5,3 @@ class ParseError(Exception): pass - - -class ArgumentError(Exception): - pass diff --git a/tests/unit/formats/test_cvat.py b/tests/unit/formats/test_cvat.py index 9826d17..7b0e476 100644 --- a/tests/unit/formats/test_cvat.py +++ b/tests/unit/formats/test_cvat.py @@ -15,7 +15,7 @@ ImageObjectDetection, SingleObjectDetection, ) -from labelformat.types import ArgumentError, ParseError +from labelformat.types import ParseError # Helper for creating temp XML files @@ -190,7 +190,7 @@ def test_save_cyclic_load_and_save( output_folder = tmp_path / "labels" CVATObjectDetectionOutput( - output_folder=output_folder, annotation_scope=annotation_scope + output_folder=output_folder, output_annotation_scope=annotation_scope.value ).save(label_input=label_input) assert output_folder.exists() From a74d7a0d96f0375277a9fe4ad39519c775a78fdf Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 12 Mar 2025 16:11:05 +0200 Subject: [PATCH 21/23] fix test and xml compare function --- tests/unit/formats/test_cvat.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/unit/formats/test_cvat.py b/tests/unit/formats/test_cvat.py index 7b0e476..a89e40b 100644 --- a/tests/unit/formats/test_cvat.py +++ b/tests/unit/formats/test_cvat.py @@ -1,5 +1,6 @@ import xml.etree.ElementTree as ET from pathlib import Path +from typing import Optional import pytest @@ -20,7 +21,7 @@ # Helper for creating temp XML files def create_xml_file(tmp_path: Path, content: str) -> Path: - xml_path = tmp_path / "labels" / "annotations.xml" + xml_path = tmp_path / "labels" / "annotations_in.xml" xml_path.parent.mkdir(parents=True, exist_ok=True) xml_path.write_text(content.strip()) return xml_path @@ -146,7 +147,11 @@ def test_get_labels_invalid_category_name(self, tmp_path: Path) -> None: def _compare_xml_elements(elem1: ET.Element, elem2: ET.Element) -> bool: """Recursively compare two XML elements for tag, attributes, and text.""" - if elem1.tag != elem2.tag or elem1.text != elem2.text: + + def normalize(text: Optional[str]) -> str: + return (text or "").strip().replace("\n", "").replace(" ", "") + + if elem1.tag != elem2.tag or normalize(elem1.text) != normalize(elem2.text): return False if elem1.attrib != elem2.attrib: @@ -169,8 +174,7 @@ class TestCVATObjectDetectionOutput: def test_save_cyclic_load_and_save( self, tmp_path: Path, annotation_scope: AnnotationScope ) -> None: - annotation = f""" - + annotation = f""" <{annotation_scope.value}> @@ -185,8 +189,8 @@ def test_save_cyclic_load_and_save( """ - xml_path = create_xml_file(tmp_path, annotation) - label_input = CVATObjectDetectionInput(xml_path) + input_xml_path = create_xml_file(tmp_path, annotation) + label_input = CVATObjectDetectionInput(input_xml_path) output_folder = tmp_path / "labels" CVATObjectDetectionOutput( @@ -196,15 +200,12 @@ def test_save_cyclic_load_and_save( assert output_folder.exists() assert output_folder.is_dir() filepaths = list(output_folder.glob("**/*.xml")) - assert len(filepaths) == 1 - path = filepaths[0] - - assert path == tmp_path / "labels" / "annotations.xml" + assert len(filepaths) == 2 - annotation = annotation.replace("\n", "") + output_xml_path = tmp_path / "labels" / "annotations.xml" # Compare XML structure. - input_tree = ET.parse(xml_path) - output_tree = ET.parse(path) + input_tree = ET.parse(input_xml_path) + output_tree = ET.parse(output_xml_path) assert _compare_xml_elements( input_tree.getroot(), output_tree.getroot() From efe6f8a88ec9abec8f73d87aa733d658d978ed94 Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 12 Mar 2025 16:23:35 +0200 Subject: [PATCH 22/23] remove default value from output-annotation-scope cli argument --- src/labelformat/formats/cvat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index b06212f..104b550 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -162,8 +162,8 @@ def add_cli_arguments(parser: ArgumentParser) -> None: ) parser.add_argument( "--output-annotation-scope", + required=True, choices=[scope.value for scope in AnnotationScope], - default="task", help="Define the annotation scope to determine the XML structure. Allowed values: " + AnnotationScope.allowed_values(), ) From 561aaba55c4e4089975429a227df109b0e9d4dcc Mon Sep 17 00:00:00 2001 From: Horatiu Almasan Date: Wed, 12 Mar 2025 16:51:42 +0200 Subject: [PATCH 23/23] add default value for output_annotation_scope --- src/labelformat/formats/cvat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/labelformat/formats/cvat.py b/src/labelformat/formats/cvat.py index 104b550..41e7214 100644 --- a/src/labelformat/formats/cvat.py +++ b/src/labelformat/formats/cvat.py @@ -162,8 +162,8 @@ def add_cli_arguments(parser: ArgumentParser) -> None: ) parser.add_argument( "--output-annotation-scope", - required=True, choices=[scope.value for scope in AnnotationScope], + default="task", help="Define the annotation scope to determine the XML structure. Allowed values: " + AnnotationScope.allowed_values(), ) @@ -171,7 +171,7 @@ def add_cli_arguments(parser: ArgumentParser) -> None: def __init__( self, output_folder: Path, - output_annotation_scope: Literal["task", "job", "project"], + output_annotation_scope: Literal["task", "job", "project"] = "task", ) -> None: try: self._annotation_scope = AnnotationScope(output_annotation_scope)