From 3a684dd2cd2dfef49b7c53d1a0b6030456a3ff64 Mon Sep 17 00:00:00 2001 From: Kevin Kim Date: Wed, 13 Jul 2022 10:39:05 -0700 Subject: [PATCH 1/2] Add video point and polyline support in annotation serialization --- labelbox/data/serialization/ndjson/label.py | 13 +-- labelbox/data/serialization/ndjson/objects.py | 58 +++++++++-- tests/data/assets/ndjson/video_import.json | 95 ++++++++++++++++++- tests/data/serialization/ndjson/test_video.py | 2 +- 4 files changed, 154 insertions(+), 14 deletions(-) diff --git a/labelbox/data/serialization/ndjson/label.py b/labelbox/data/serialization/ndjson/label.py index 529d711a7..ac6916dd0 100644 --- a/labelbox/data/serialization/ndjson/label.py +++ b/labelbox/data/serialization/ndjson/label.py @@ -1,6 +1,6 @@ from itertools import groupby from operator import itemgetter -from typing import Dict, Generator, List, Tuple, Union +from typing import Any, Dict, Generator, List, Tuple, Union from collections import defaultdict import warnings @@ -51,7 +51,6 @@ def _generate_annotations( if isinstance(annotation, NDSegments): annots.extend( NDSegments.to_common(annotation, annotation.schema_id)) - elif isinstance(annotation, NDObjectType.__args__): annots.append(NDObject.to_common(annotation)) elif isinstance(annotation, NDClassificationType.__args__): @@ -62,12 +61,15 @@ def _generate_annotations( else: raise TypeError( f"Unsupported annotation. {type(annotation)}") - data = self._infer_media_type(annotations)(uid=data_row_id) + data = self._infer_media_type(annots)(uid=data_row_id) yield Label(annotations=annots, data=data) def _infer_media_type( - self, annotations: List[Union[NDObjectType, NDClassificationType]] - ) -> Union[TextEntity, TextData, ImageData]: + self, annotations: List[Union[TextEntity, VideoClassificationAnnotation, + VideoObjectAnnotation, ObjectAnnotation, + ClassificationAnnotation, ScalarMetric, + ConfusionMatrixMetric]] + ) -> Union[TextData, VideoData, ImageData]: types = {type(annotation) for annotation in annotations} if TextEntity in types: return TextData @@ -100,7 +102,6 @@ def _create_video_annotations( for annotation_group in video_annotations.values(): consecutive_frames = cls._get_consecutive_frames( sorted([annotation.frame for annotation in annotation_group])) - if isinstance(annotation_group[0], VideoClassificationAnnotation): annotation = annotation_group[0] frames_data = [] diff --git a/labelbox/data/serialization/ndjson/objects.py b/labelbox/data/serialization/ndjson/objects.py index ccf1e69f8..fadf95fc5 100644 --- a/labelbox/data/serialization/ndjson/objects.py +++ b/labelbox/data/serialization/ndjson/objects.py @@ -60,6 +60,21 @@ def from_common(cls, point: Point, classifications=classifications) +class NDFramePoint(VideoSupported): + point: _Point + + def to_common(self, feature_schema_id: Cuid) -> VideoObjectAnnotation: + return VideoObjectAnnotation(frame=self.frame, + keyframe=True, + feature_schema_id=feature_schema_id, + value=Point(x=self.point.x, + y=self.point.y)) + + @classmethod + def from_common(cls, frame: int, point: Point): + return cls(frame=frame, point=_Point(x=point.x, y=point.y)) + + class NDLine(NDBaseObject): line: List[_Point] @@ -81,6 +96,25 @@ def from_common(cls, line: Line, classifications=classifications) +class NDFrameLine(VideoSupported): + line: List[_Point] + + def to_common(self, feature_schema_id: Cuid) -> VideoObjectAnnotation: + return VideoObjectAnnotation( + frame=self.frame, + keyframe=True, + feature_schema_id=feature_schema_id, + value=Line(points=[Point(x=pt.x, y=pt.y) for pt in self.line])) + + @classmethod + def from_common(cls, frame: int, line: Line): + return cls(frame=frame, + line=[{ + 'x': pt.x, + 'y': pt.y + } for pt in line.points]) + + class NDPolygon(NDBaseObject): polygon: List[_Point] @@ -147,18 +181,29 @@ def from_common(cls, frame: int, rectangle: Rectangle): class NDSegment(BaseModel): - keyframes: List[NDFrameRectangle] + keyframes: List[Union[NDFrameRectangle, NDFramePoint, NDFrameLine]] @staticmethod def lookup_segment_object_type(segment: List) -> "NDFrameObjectType": """Used for determining which object type the annotation contains returns the object type""" - result = {Rectangle: NDFrameRectangle}.get(type(segment[0].value)) + result = { + Rectangle: NDFrameRectangle, + Point: NDFramePoint, + Line: NDFrameLine, + }.get(type(segment[0].value)) return result - def to_common(self, feature_schema_id: Cuid): + @staticmethod + def segment_with_uuid(keyframe: Union[NDFrameRectangle, NDFramePoint, + NDFrameLine], uuid: str): + keyframe.extra = {'uuid': uuid} + return keyframe + + def to_common(self, feature_schema_id: Cuid, uuid: str): return [ - keyframe.to_common(feature_schema_id) for keyframe in self.keyframes + self.segment_with_uuid(keyframe.to_common(feature_schema_id), uuid) + for keyframe in self.keyframes ] @classmethod @@ -178,7 +223,8 @@ class NDSegments(NDBaseObject): def to_common(self, feature_schema_id: Cuid): result = [] for segment in self.segments: - result.extend(NDSegment.to_common(segment, feature_schema_id)) + result.extend( + NDSegment.to_common(segment, feature_schema_id, self.uuid)) return result @classmethod @@ -330,4 +376,4 @@ def lookup_object( NDObjectType = Union[NDLine, NDPolygon, NDPoint, NDRectangle, NDMask, NDTextEntity] -NDFrameObjectType = NDFrameRectangle +NDFrameObjectType = NDFrameRectangle, NDFramePoint, NDFrameLine diff --git a/tests/data/assets/ndjson/video_import.json b/tests/data/assets/ndjson/video_import.json index ec48027dd..0e1bd11e2 100644 --- a/tests/data/assets/ndjson/video_import.json +++ b/tests/data/assets/ndjson/video_import.json @@ -1 +1,94 @@ -[{"answer": {"schemaId": "ckrb1sfl8099g0y91cxbd5ftb"}, "schemaId": "ckrb1sfjx099a0y914hl319ie", "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, "uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673", "frames": [{"start": 30, "end": 35}, {"start": 50, "end": 51}]}, {"answer": [{"schemaId": "ckrb1sfl8099e0y919v260awv"}], "schemaId": "ckrb1sfkn099c0y910wbo0p1a", "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, "uuid": "d009925d-91a3-4f67-abd9-753453f5a584", "frames": [{"start": 0, "end": 5}]}, {"answer": "a value", "schemaId": "ckrb1sfkn099c0y910wbo0p1a", "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, "uuid": "d009925d-91a3-4f67-abd9-753453f5a584"}] \ No newline at end of file +[ + { + "answer": {"schemaId": "ckrb1sfl8099g0y91cxbd5ftb"}, + "schemaId": "ckrb1sfjx099a0y914hl319ie", + "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, + "uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673", + "frames": [{"start": 30, "end": 35}, {"start": 50, "end": 51}] + }, + { + "answer": [{"schemaId": "ckrb1sfl8099e0y919v260awv"}], + "schemaId": "ckrb1sfkn099c0y910wbo0p1a", + "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, + "uuid": "d009925d-91a3-4f67-abd9-753453f5a584", + "frames": [{"start": 0, "end": 5}] + }, + { + "answer": "a value", + "schemaId": "ckrb1sfkn099c0y910wbo0p1a", + "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, + "uuid": "d009925d-91a3-4f67-abd9-753453f5a584" + }, + { + "classifications": [], + "schemaId": "cl5islwg200gfci6g0oitaypu", + "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, + "uuid": "6f7c835a-0139-4896-b73f-66a6baa89e94", + "segments": [ + { + "keyframes": [ + { + "frame": 1, + "line": [{"x": 10.0, "y": 10.0}, {"x": 100.0, "y": 100.0}, {"x": 50.0, "y": 30.0}] + } + ] + }, + { + "keyframes": [ + { + "frame": 5, + "line": [{"x": 100.0, "y": 10.0}, {"x": 50.0, "y": 100.0}, {"x": 50.0, "y": 30.0}] + } + ] + } + ] + }, + { + "classifications": [], + "schemaId": "cl5it7ktp00i5ci6gf80b1ysd", + "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, + "uuid": "f963be22-227b-4efe-9be4-2738ed822216", + "segments": [ + { + "keyframes": [ + { + "frame": 1, + "point": {"x": 10.0, "y": 10.0} + } + ] + }, + { + "keyframes": [ + { + "frame": 5, + "point": {"x": 50.0, "y": 50.0} + } + ] + } + ] + }, + { + "classifications": [], + "schemaId": "cl5iw0roz00lwci6g5jni62vs", + "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, + "uuid": "13b2ee0e-2355-4336-8b83-d74d09e3b1e7", + "segments": [ + { + "keyframes": [ + { + "frame": 1, + "bbox": {"top": 10.0, "left": 5.0, "height": 100.0, "width": 150.0} + } + ] + }, + { + "keyframes": [ + { + "frame": 5, + "bbox": {"top": 300.0, "left": 200.0, "height": 400.0, "width": 150.0} + } + ] + } + ] + } +] \ No newline at end of file diff --git a/tests/data/serialization/ndjson/test_video.py b/tests/data/serialization/ndjson/test_video.py index 67ea3e2a5..01eda7175 100644 --- a/tests/data/serialization/ndjson/test_video.py +++ b/tests/data/serialization/ndjson/test_video.py @@ -9,4 +9,4 @@ def test_video(): res = NDJsonConverter.deserialize(data).as_list() res = list(NDJsonConverter.serialize(res)) - assert res == [data[2], data[0], data[1]] + assert res == [data[2], data[0], data[1], data[3], data[4], data[5]] From 8f6d5306f3c11280cecb9080f518976acb17789d Mon Sep 17 00:00:00 2001 From: Kevin Kim Date: Wed, 13 Jul 2022 11:09:48 -0700 Subject: [PATCH 2/2] Remove unused import --- labelbox/data/serialization/ndjson/label.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelbox/data/serialization/ndjson/label.py b/labelbox/data/serialization/ndjson/label.py index ac6916dd0..f62c8af9d 100644 --- a/labelbox/data/serialization/ndjson/label.py +++ b/labelbox/data/serialization/ndjson/label.py @@ -1,6 +1,6 @@ from itertools import groupby from operator import itemgetter -from typing import Any, Dict, Generator, List, Tuple, Union +from typing import Dict, Generator, List, Tuple, Union from collections import defaultdict import warnings