From 3ba8c7e298c38bf08de4333e95a59abd6752cc01 Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:19:27 +0000 Subject: [PATCH] Revert "Resolved precision bug & some linting (#751)" This reverts commit 254d05adeb417596c2cd4bc1cc06a8d227dbf6d1. --- darwin/dataset/local_dataset.py | 46 ++--- darwin/exporter/formats/darwin_1_0.py | 42 ++--- darwin/importer/formats/csv_tags_video.py | 4 +- darwin/importer/importer.py | 4 +- darwin/torch/dataset.py | 32 +--- darwin/utils/utils.py | 209 ++++++---------------- 6 files changed, 83 insertions(+), 254 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 435062276..b8c48a7ea 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -80,9 +80,7 @@ def __init__( self.original_annotations_path: Optional[List[Path]] = None self.keep_empty_annotations = keep_empty_annotations - release_path, annotations_dir, images_dir = self._initial_setup( - dataset_path, release_name - ) + release_path, annotations_dir, images_dir = self._initial_setup(dataset_path, release_name) self._validate_inputs(partition, split_type, annotation_type) # Get the list of classes @@ -122,9 +120,7 @@ def _validate_inputs(self, partition, split_type, annotation_type): if split_type not in ["random", "stratified"]: raise ValueError("split_type should be either 'random', 'stratified'") if annotation_type not in ["tag", "polygon", "bounding_box"]: - raise ValueError( - "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" - ) + raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") def _setup_annotations_and_images( self, @@ -152,9 +148,7 @@ def _setup_annotations_and_images( darwin_json, images_dir, with_folders, json_version, annotation_filepath ) if image_path.exists(): - if not keep_empty_annotations and is_stream_list_empty( - darwin_json["annotations"] - ): + if not keep_empty_annotations and is_stream_list_empty(darwin_json["annotations"]): continue self.images_path.append(image_path) self.annotations_path.append(annotation_filepath) @@ -221,9 +215,7 @@ def get_height_and_width(self, index: int) -> Tuple[float, float]: parsed = parse_darwin_json(self.annotations_path[index], index) return parsed.image_height, parsed.image_width - def extend( - self, dataset: "LocalDataset", extend_classes: bool = False - ) -> "LocalDataset": + def extend(self, dataset: "LocalDataset", extend_classes: bool = False) -> "LocalDataset": """ Extends the current dataset with another one. @@ -318,10 +310,7 @@ def parse_json(self, index: int) -> Dict[str, Any]: # Filter out unused classes and annotations of a different type if self.classes is not None: annotations = [ - a - for a in annotations - if a.annotation_class.name in self.classes - and self.annotation_type_supported(a) + a for a in annotations if a.annotation_class.name in self.classes and self.annotation_type_supported(a) ] return { "image_id": index, @@ -338,20 +327,15 @@ def annotation_type_supported(self, annotation) -> bool: elif self.annotation_type == "bounding_box": is_bounding_box = annotation_type == "bounding_box" is_supported_polygon = ( - annotation_type in ["polygon", "complex_polygon"] - and "bounding_box" in annotation.data + annotation_type in ["polygon", "complex_polygon"] and "bounding_box" in annotation.data ) return is_bounding_box or is_supported_polygon elif self.annotation_type == "polygon": return annotation_type in ["polygon", "complex_polygon"] else: - raise ValueError( - "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" - ) + raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") - def measure_mean_std( - self, multi_threaded: bool = True - ) -> Tuple[np.ndarray, np.ndarray]: + def measure_mean_std(self, multi_threaded: bool = True) -> Tuple[np.ndarray, np.ndarray]: """ Computes mean and std of trained images, given the train loader. @@ -374,9 +358,7 @@ def measure_mean_std( results = pool.map(self._return_mean, self.images_path) mean = np.sum(np.array(results), axis=0) / len(self.images_path) # Online image_classification deviation - results = pool.starmap( - self._return_std, [[item, mean] for item in self.images_path] - ) + results = pool.starmap(self._return_std, [[item, mean] for item in self.images_path]) std_sum = np.sum(np.array([item[0] for item in results]), axis=0) total_pixel_count = np.sum(np.array([item[1] for item in results])) std = np.sqrt(std_sum / total_pixel_count) @@ -422,20 +404,14 @@ def _compute_weights(labels: List[int]) -> np.ndarray: @staticmethod def _return_mean(image_path: Path) -> np.ndarray: img = np.array(load_pil_image(image_path)) - mean = np.array( - [np.mean(img[:, :, 0]), np.mean(img[:, :, 1]), np.mean(img[:, :, 2])] - ) + mean = np.array([np.mean(img[:, :, 0]), np.mean(img[:, :, 1]), np.mean(img[:, :, 2])]) return mean / 255.0 # Loads an image with OpenCV and returns the channel wise std of the image. @staticmethod def _return_std(image_path: Path, mean: np.ndarray) -> Tuple[np.ndarray, float]: img = np.array(load_pil_image(image_path)) / 255.0 - m2 = np.square( - np.array( - [img[:, :, 0] - mean[0], img[:, :, 1] - mean[1], img[:, :, 2] - mean[2]] - ) - ) + m2 = np.square(np.array([img[:, :, 0] - mean[0], img[:, :, 1] - mean[1], img[:, :, 2] - mean[2]])) return np.sum(np.sum(m2, axis=1), 1), m2.size / 3.0 def __getitem__(self, index: int): diff --git a/darwin/exporter/formats/darwin_1_0.py b/darwin/exporter/formats/darwin_1_0.py index 4adc6b3ad..f78af61e8 100644 --- a/darwin/exporter/formats/darwin_1_0.py +++ b/darwin/exporter/formats/darwin_1_0.py @@ -45,23 +45,17 @@ def _export_file(annotation_file: AnnotationFile, _: int, output_dir: Path) -> N try: output: DictFreeForm = _build_json(annotation_file) except Exception as e: - raise ExportException_CouldNotBuildOutput( - f"Could not build output for {annotation_file.path}" - ) from e + raise ExportException_CouldNotBuildOutput(f"Could not build output for {annotation_file.path}") from e try: with open(output_file_path, "w") as f: op = json.dumps( output, - option=json.OPT_INDENT_2 - | json.OPT_SERIALIZE_NUMPY - | json.OPT_NON_STR_KEYS, + option=json.OPT_INDENT_2 | json.OPT_SERIALIZE_NUMPY | json.OPT_NON_STR_KEYS, ).decode("utf-8") f.write(op) except Exception as e: - raise ExportException_CouldNotWriteFile( - f"Could not write output for {annotation_file.path}" - ) from e + raise ExportException_CouldNotWriteFile(f"Could not write output for {annotation_file.path}") from e def _build_json(annotation_file: AnnotationFile) -> DictFreeForm: @@ -136,17 +130,11 @@ def _build_sub_annotation(sub: SubAnnotation) -> DictFreeForm: def _build_authorship(annotation: Union[VideoAnnotation, Annotation]) -> DictFreeForm: annotators = {} if annotation.annotators: - annotators = { - "annotators": [ - _build_author(annotator) for annotator in annotation.annotators - ] - } + annotators = {"annotators": [_build_author(annotator) for annotator in annotation.annotators]} reviewers = {} if annotation.reviewers: - reviewers = { - "annotators": [_build_author(reviewer) for reviewer in annotation.reviewers] - } + reviewers = {"annotators": [_build_author(reviewer) for reviewer in annotation.reviewers]} return {**annotators, **reviewers} @@ -155,9 +143,7 @@ def _build_video_annotation(annotation: VideoAnnotation) -> DictFreeForm: return { **annotation.get_data( only_keyframes=False, - post_processing=lambda annotation, _: _build_image_annotation( - annotation, skip_slots=True - ), + post_processing=lambda annotation, _: _build_image_annotation(annotation, skip_slots=True), ), "name": annotation.annotation_class.name, "slot_names": annotation.slot_names, @@ -165,9 +151,7 @@ def _build_video_annotation(annotation: VideoAnnotation) -> DictFreeForm: } -def _build_image_annotation( - annotation: Annotation, skip_slots: bool = False -) -> DictFreeForm: +def _build_image_annotation(annotation: Annotation, skip_slots: bool = False) -> DictFreeForm: json_subs = {} for sub in annotation.subs: json_subs.update(_build_sub_annotation(sub)) @@ -185,9 +169,7 @@ def _build_image_annotation( return {**base_json, "slot_names": annotation.slot_names} -def _build_legacy_annotation_data( - annotation_class: AnnotationClass, data: DictFreeForm -) -> DictFreeForm: +def _build_legacy_annotation_data(annotation_class: AnnotationClass, data: DictFreeForm) -> DictFreeForm: v1_data = {} polygon_annotation_mappings = {"complex_polygon": "paths", "polygon": "path"} @@ -250,9 +232,7 @@ def build_image_annotation(annotation_file: AnnotationFile) -> Dict[str, Any]: annotations: List[Dict[str, Any]] = [] for annotation in annotation_file.annotations: payload = { - annotation.annotation_class.annotation_type: _build_annotation_data( - annotation - ), + annotation.annotation_class.annotation_type: _build_annotation_data(annotation), "name": annotation.annotation_class.name, } @@ -280,8 +260,6 @@ def _build_annotation_data(annotation: Annotation) -> Dict[str, Any]: return {"path": annotation.data["paths"]} if annotation.annotation_class.annotation_type == "polygon": - return dict( - filter(lambda item: item[0] != "bounding_box", annotation.data.items()) - ) + return dict(filter(lambda item: item[0] != "bounding_box", annotation.data.items())) return dict(annotation.data) diff --git a/darwin/importer/formats/csv_tags_video.py b/darwin/importer/formats/csv_tags_video.py index fd9cdcb3b..a6885ac0c 100644 --- a/darwin/importer/formats/csv_tags_video.py +++ b/darwin/importer/formats/csv_tags_video.py @@ -51,7 +51,9 @@ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]: file_annotation_map[filename].append(annotation) for filename in file_annotation_map: annotations = file_annotation_map[filename] - annotation_classes = {annotation.annotation_class for annotation in annotations} + annotation_classes = { + annotation.annotation_class for annotation in annotations + } filename_path = Path(filename) remote_path = str(filename_path.parent) if not remote_path.startswith("/"): diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index 7bc12edc6..d2f02f807 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -603,9 +603,7 @@ def _warn_unsupported_annotations(parsed_files: List[AnnotationFile]) -> None: if annotation.annotation_class.annotation_type in UNSUPPORTED_CLASSES: skipped_annotations.append(annotation) if len(skipped_annotations) > 0: - types = { - c.annotation_class.annotation_type for c in skipped_annotations - } # noqa: C417 + types = {c.annotation_class.annotation_type for c in skipped_annotations} # noqa: C417 console.print( f"Import of annotation class types '{', '.join(types)}' is not yet supported. Skipping {len(skipped_annotations)} " + "annotations from '{parsed_file.full_path}'.\n", diff --git a/darwin/torch/dataset.py b/darwin/torch/dataset.py index b98592e0c..b17a21238 100644 --- a/darwin/torch/dataset.py +++ b/darwin/torch/dataset.py @@ -99,9 +99,7 @@ class ClassificationDataset(LocalDataset): be composed via torchvision. """ - def __init__( - self, transform: Optional[Union[Callable, List]] = None, **kwargs - ) -> None: + def __init__(self, transform: Optional[Union[Callable, List]] = None, **kwargs) -> None: super().__init__(annotation_type="tag", **kwargs) if transform is not None and isinstance(transform, list): @@ -154,11 +152,7 @@ def get_target(self, index: int) -> Tensor: data = self.parse_json(index) annotations = data.pop("annotations") - tags = [ - a.annotation_class.name - for a in annotations - if a.annotation_class.annotation_type == "tag" - ] + tags = [a.annotation_class.name for a in annotations if a.annotation_class.annotation_type == "tag"] if not self.is_multi_label: # Binary or multiclass must have a label per image @@ -182,11 +176,7 @@ def check_if_multi_label(self) -> None: for idx in range(len(self)): target = self.parse_json(idx) annotations = target.pop("annotations") - tags = [ - a.annotation_class.name - for a in annotations - if a.annotation_class.annotation_type == "tag" - ] + tags = [a.annotation_class.name for a in annotations if a.annotation_class.annotation_type == "tag"] if len(tags) > 1: self.is_multi_label = True @@ -334,15 +324,12 @@ def get_target(self, index: int) -> Dict[str, Any]: path_key = "paths" if path_key not in annotation.data: - print( - f"Warning: missing polygon in annotation {self.annotations_path[index]}" - ) + print(f"Warning: missing polygon in annotation {self.annotations_path[index]}") # Extract the sequences of coordinates from the polygon annotation sequences = convert_polygons_to_sequences( annotation.data[path_key], height=target["height"], width=target["width"], - rounding=False, ) # Compute the bbox of the polygon x_coords = [s[0::2] for s in sequences] @@ -366,12 +353,7 @@ def get_target(self, index: int) -> Dict[str, Any]: # Compute the area of the polygon # TODO fix with addictive/subtractive paths in complex polygons - poly_area: float = np.sum( - [ - polygon_area(x_coord, y_coord) - for x_coord, y_coord in zip(x_coords, y_coords) - ] - ) + poly_area: float = np.sum([polygon_area(x_coord, y_coord) for x_coord, y_coord in zip(x_coords, y_coords)]) # Create and append the new entry for this annotation annotations.append( @@ -423,9 +405,7 @@ class SemanticSegmentationDataset(LocalDataset): Object used to convert polygons to semantic masks. """ - def __init__( - self, transform: Optional[Union[List[Callable], Callable]] = None, **kwargs - ): + def __init__(self, transform: Optional[Union[List[Callable], Callable]] = None, **kwargs): super().__init__(annotation_type="polygon", **kwargs) if "__background__" not in self.classes: self.classes.insert(0, "__background__") diff --git a/darwin/utils/utils.py b/darwin/utils/utils.py index c2e2b6f2f..02ce6d3e8 100644 --- a/darwin/utils/utils.py +++ b/darwin/utils/utils.py @@ -214,9 +214,7 @@ def is_project_dir(project_path: Path) -> bool: return (project_path / "releases").exists() and (project_path / "images").exists() -def get_progress_bar( - array: List[dt.AnnotationFile], description: Optional[str] = None -) -> Iterable[ProgressType]: +def get_progress_bar(array: List[dt.AnnotationFile], description: Optional[str] = None) -> Iterable[ProgressType]: """ Get a rich a progress bar for the given list of annotation files. @@ -360,9 +358,7 @@ def persist_client_configuration( api_key=team_config.api_key, datasets_dir=team_config.datasets_dir, ) - config.set_global( - api_endpoint=client.url, base_url=client.base_url, default_team=default_team - ) + config.set_global(api_endpoint=client.url, base_url=client.base_url, default_team=default_team) return config @@ -419,9 +415,7 @@ def attempt_decode(path: Path) -> dict: return data except Exception: continue - raise UnrecognizableFileEncoding( - f"Unable to load file {path} with any encodings: {encodings}" - ) + raise UnrecognizableFileEncoding(f"Unable to load file {path} with any encodings: {encodings}") def load_data_from_file(path: Path) -> Tuple[dict, dt.AnnotationFileVersion]: @@ -430,9 +424,7 @@ def load_data_from_file(path: Path) -> Tuple[dict, dt.AnnotationFileVersion]: return data, version -def parse_darwin_json( - path: Path, count: Optional[int] = None -) -> Optional[dt.AnnotationFile]: +def parse_darwin_json(path: Path, count: Optional[int] = None) -> Optional[dt.AnnotationFile]: """ Parses the given JSON file in v7's darwin proprietary format. Works for images, split frame videos (treated as images) and playback videos. @@ -490,7 +482,7 @@ def stream_darwin_json(path: Path) -> PersistentStreamingJSONObject: with path.open() as infile: return json_stream.load(infile, persistent=True) - + def get_image_path_from_stream( darwin_json: PersistentStreamingJSONObject, @@ -565,7 +557,7 @@ def get_darwin_json_version(annotations_dir: Path) -> str: data_str = file.read() data = json.loads(data_str) return "2.0" if "version" in data and data["version"] == "2.0" else "1.0" - + def is_stream_list_empty(json_list: PersistentStreamingJSONList) -> bool: try: @@ -579,15 +571,9 @@ def is_stream_list_empty(json_list: PersistentStreamingJSONList) -> bool: def _parse_darwin_v2(path: Path, data: Dict[str, Any]) -> dt.AnnotationFile: item = data["item"] item_source = item.get("source_info", {}) - slots: List[dt.Slot] = list( - filter(None, map(_parse_darwin_slot, item.get("slots", []))) - ) - annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations( - data - ) - annotation_classes: Set[dt.AnnotationClass] = { - annotation.annotation_class for annotation in annotations - } + slots: List[dt.Slot] = list(filter(None, map(_parse_darwin_slot, item.get("slots", [])))) + annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations(data) + annotation_classes: Set[dt.AnnotationClass] = {annotation.annotation_class for annotation in annotations} if len(slots) == 0: annotation_file = dt.AnnotationFile( @@ -595,9 +581,7 @@ def _parse_darwin_v2(path: Path, data: Dict[str, Any]) -> dt.AnnotationFile: path=path, filename=item["name"], item_id=item.get("source_info", {}).get("item_id", None), - dataset_name=item.get("source_info", {}) - .get("dataset", {}) - .get("name", None), + dataset_name=item.get("source_info", {}).get("dataset", {}).get("name", None), annotation_classes=annotation_classes, annotations=annotations, is_video=False, @@ -618,17 +602,13 @@ def _parse_darwin_v2(path: Path, data: Dict[str, Any]) -> dt.AnnotationFile: path=path, filename=item["name"], item_id=item.get("source_info", {}).get("item_id", None), - dataset_name=item.get("source_info", {}) - .get("dataset", {}) - .get("name", None), + dataset_name=item.get("source_info", {}).get("dataset", {}).get("name", None), annotation_classes=annotation_classes, annotations=annotations, is_video=slot.frame_urls is not None or slot.frame_manifest is not None, image_width=slot.width, image_height=slot.height, - image_url=None - if len(slot.source_files or []) == 0 - else slot.source_files[0]["url"], + image_url=None if len(slot.source_files or []) == 0 else slot.source_files[0]["url"], image_thumbnail_url=slot.thumbnail_url, workview_url=item_source.get("workview_url", None), seq=0, @@ -658,15 +638,9 @@ def _parse_darwin_slot(data: Dict[str, Any]) -> dt.Slot: ) -def _parse_darwin_image( - path: Path, data: Dict[str, Any], count: Optional[int] -) -> dt.AnnotationFile: - annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations( - data - ) - annotation_classes: Set[dt.AnnotationClass] = { - annotation.annotation_class for annotation in annotations - } +def _parse_darwin_image(path: Path, data: Dict[str, Any], count: Optional[int]) -> dt.AnnotationFile: + annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations(data) + annotation_classes: Set[dt.AnnotationClass] = {annotation.annotation_class for annotation in annotations} slot = dt.Slot( name=None, @@ -703,20 +677,12 @@ def _parse_darwin_image( return annotation_file -def _parse_darwin_video( - path: Path, data: Dict[str, Any], count: Optional[int] -) -> dt.AnnotationFile: - annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations( - data - ) - annotation_classes: Set[dt.AnnotationClass] = { - annotation.annotation_class for annotation in annotations - } +def _parse_darwin_video(path: Path, data: Dict[str, Any], count: Optional[int]) -> dt.AnnotationFile: + annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations(data) + annotation_classes: Set[dt.AnnotationClass] = {annotation.annotation_class for annotation in annotations} if "width" not in data["image"] or "height" not in data["image"]: - raise OutdatedDarwinJSONFormat( - "Missing width/height in video, please re-export" - ) + raise OutdatedDarwinJSONFormat("Missing width/height in video, please re-export") slot = dt.Slot( name=None, @@ -762,41 +728,23 @@ def _parse_darwin_annotation(annotation: Dict[str, Any]) -> Optional[dt.Annotati main_annotation: Optional[dt.Annotation] = None # Darwin JSON 2.0 representation of complex polygons - if ( - "polygon" in annotation - and "paths" in annotation["polygon"] - and len(annotation["polygon"]["paths"]) > 1 - ): + if "polygon" in annotation and "paths" in annotation["polygon"] and len(annotation["polygon"]["paths"]) > 1: bounding_box = annotation.get("bounding_box") paths = annotation["polygon"]["paths"] - main_annotation = dt.make_complex_polygon( - name, paths, bounding_box, slot_names=slot_names - ) + main_annotation = dt.make_complex_polygon(name, paths, bounding_box, slot_names=slot_names) # Darwin JSON 2.0 representation of simple polygons - elif ( - "polygon" in annotation - and "paths" in annotation["polygon"] - and len(annotation["polygon"]["paths"]) == 1 - ): + elif "polygon" in annotation and "paths" in annotation["polygon"] and len(annotation["polygon"]["paths"]) == 1: bounding_box = annotation.get("bounding_box") paths = annotation["polygon"]["paths"] - main_annotation = dt.make_polygon( - name, paths[0], bounding_box, slot_names=slot_names - ) + main_annotation = dt.make_polygon(name, paths[0], bounding_box, slot_names=slot_names) # Darwin JSON 1.0 representation of complex and simple polygons elif "polygon" in annotation: bounding_box = annotation.get("bounding_box") if "additional_paths" in annotation["polygon"]: - paths = [annotation["polygon"]["path"]] + annotation["polygon"][ - "additional_paths" - ] - main_annotation = dt.make_complex_polygon( - name, paths, bounding_box, slot_names=slot_names - ) + paths = [annotation["polygon"]["path"]] + annotation["polygon"]["additional_paths"] + main_annotation = dt.make_complex_polygon(name, paths, bounding_box, slot_names=slot_names) else: - main_annotation = dt.make_polygon( - name, annotation["polygon"]["path"], bounding_box, slot_names=slot_names - ) + main_annotation = dt.make_polygon(name, annotation["polygon"]["path"], bounding_box, slot_names=slot_names) # Darwin JSON 1.0 representation of complex polygons elif "complex_polygon" in annotation: bounding_box = annotation.get("bounding_box") @@ -808,9 +756,7 @@ def _parse_darwin_annotation(annotation: Dict[str, Any]) -> Optional[dt.Annotati if "additional_paths" in annotation["complex_polygon"]: paths.extend(annotation["complex_polygon"]["additional_paths"]) - main_annotation = dt.make_complex_polygon( - name, paths, bounding_box, slot_names=slot_names - ) + main_annotation = dt.make_complex_polygon(name, paths, bounding_box, slot_names=slot_names) elif "bounding_box" in annotation: bounding_box = annotation["bounding_box"] main_annotation = dt.make_bounding_box( @@ -824,9 +770,7 @@ def _parse_darwin_annotation(annotation: Dict[str, Any]) -> Optional[dt.Annotati elif "tag" in annotation: main_annotation = dt.make_tag(name, slot_names=slot_names) elif "line" in annotation: - main_annotation = dt.make_line( - name, annotation["line"]["path"], slot_names=slot_names - ) + main_annotation = dt.make_line(name, annotation["line"]["path"], slot_names=slot_names) elif "keypoint" in annotation: main_annotation = dt.make_keypoint( name, @@ -835,17 +779,11 @@ def _parse_darwin_annotation(annotation: Dict[str, Any]) -> Optional[dt.Annotati slot_names=slot_names, ) elif "ellipse" in annotation: - main_annotation = dt.make_ellipse( - name, annotation["ellipse"], slot_names=slot_names - ) + main_annotation = dt.make_ellipse(name, annotation["ellipse"], slot_names=slot_names) elif "cuboid" in annotation: - main_annotation = dt.make_cuboid( - name, annotation["cuboid"], slot_names=slot_names - ) + main_annotation = dt.make_cuboid(name, annotation["cuboid"], slot_names=slot_names) elif "skeleton" in annotation: - main_annotation = dt.make_skeleton( - name, annotation["skeleton"]["nodes"], slot_names=slot_names - ) + main_annotation = dt.make_skeleton(name, annotation["skeleton"]["nodes"], slot_names=slot_names) elif "table" in annotation: main_annotation = dt.make_table( name, @@ -854,9 +792,7 @@ def _parse_darwin_annotation(annotation: Dict[str, Any]) -> Optional[dt.Annotati slot_names=slot_names, ) elif "string" in annotation: - main_annotation = dt.make_string( - name, annotation["string"]["sources"], slot_names=slot_names - ) + main_annotation = dt.make_string(name, annotation["string"]["sources"], slot_names=slot_names) elif "graph" in annotation: main_annotation = dt.make_graph( name, @@ -883,29 +819,19 @@ def _parse_darwin_annotation(annotation: Dict[str, Any]) -> Optional[dt.Annotati if "id" in annotation: main_annotation.id = annotation["id"] if "instance_id" in annotation: - main_annotation.subs.append( - dt.make_instance_id(annotation["instance_id"]["value"]) - ) + main_annotation.subs.append(dt.make_instance_id(annotation["instance_id"]["value"])) if "attributes" in annotation: main_annotation.subs.append(dt.make_attributes(annotation["attributes"])) if "text" in annotation: main_annotation.subs.append(dt.make_text(annotation["text"]["text"])) if "inference" in annotation: - main_annotation.subs.append( - dt.make_opaque_sub("inference", annotation["inference"]) - ) + main_annotation.subs.append(dt.make_opaque_sub("inference", annotation["inference"])) if "directional_vector" in annotation: - main_annotation.subs.append( - dt.make_opaque_sub("directional_vector", annotation["directional_vector"]) - ) + main_annotation.subs.append(dt.make_opaque_sub("directional_vector", annotation["directional_vector"])) if "measures" in annotation: - main_annotation.subs.append( - dt.make_opaque_sub("measures", annotation["measures"]) - ) + main_annotation.subs.append(dt.make_opaque_sub("measures", annotation["measures"])) if "auto_annotate" in annotation: - main_annotation.subs.append( - dt.make_opaque_sub("auto_annotate", annotation["auto_annotate"]) - ) + main_annotation.subs.append(dt.make_opaque_sub("auto_annotate", annotation["auto_annotate"])) if annotation.get("annotators") is not None: main_annotation.annotators = _parse_annotators(annotation["annotators"]) @@ -959,9 +885,7 @@ def _parse_darwin_raster_annotation(annotation: dict) -> Optional[dt.Annotation] slot_names: Optional[List[str]] = parse_slot_names(annotation) if not id or not name or not raster_layer: - raise ValueError( - "Raster annotation must have an 'id', 'name' and 'raster_layer' field" - ) + raise ValueError("Raster annotation must have an 'id', 'name' and 'raster_layer' field") dense_rle, mask_annotation_ids_mapping, total_pixels = ( raster_layer.get("dense_rle", None), @@ -1012,14 +936,9 @@ def _parse_darwin_mask_annotation(annotation: dict) -> Optional[dt.Annotation]: def _parse_annotators(annotators: List[Dict[str, Any]]) -> List[dt.AnnotationAuthor]: if not (hasattr(annotators, "full_name") or not hasattr(annotators, "email")): - raise AttributeError( - "JSON file must contain annotators with 'full_name' and 'email' fields" - ) + raise AttributeError("JSON file must contain annotators with 'full_name' and 'email' fields") - return [ - dt.AnnotationAuthor(annotator["full_name"], annotator["email"]) - for annotator in annotators - ] + return [dt.AnnotationAuthor(annotator["full_name"], annotator["email"]) for annotator in annotators] def split_video_annotation(annotation: dt.AnnotationFile) -> List[dt.AnnotationFile]: @@ -1055,13 +974,9 @@ def split_video_annotation(annotation: dt.AnnotationFile) -> List[dt.AnnotationF frame_annotations = [] for i, frame_url in enumerate(urls): annotations = [ - a.frames[i] - for a in annotation.annotations - if isinstance(a, dt.VideoAnnotation) and i in a.frames + a.frames[i] for a in annotation.annotations if isinstance(a, dt.VideoAnnotation) and i in a.frames ] - annotation_classes: Set[dt.AnnotationClass] = { - annotation.annotation_class for annotation in annotations - } + annotation_classes: Set[dt.AnnotationClass] = {annotation.annotation_class for annotation in annotations} filename: str = f"{Path(annotation.filename).stem}/{i:07d}.png" frame_annotations.append( dt.AnnotationFile( @@ -1147,9 +1062,7 @@ def convert_polygons_to_sequences( else: list_polygons = cast(List[dt.Polygon], [polygons]) - if not isinstance(list_polygons[0], list) or not isinstance( - list_polygons[0][0], dict - ): + if not isinstance(list_polygons[0], list) or not isinstance(list_polygons[0][0], dict): raise ValueError("Unknown input format") sequences: List[List[Union[int, float]]] = [] @@ -1290,9 +1203,7 @@ def convert_bounding_box_to_xyxy(box: dt.BoundingBox) -> List[float]: return [box["x"], box["y"], x2, y2] -def convert_polygons_to_mask( - polygons: List, height: int, width: int, value: Optional[int] = 1 -) -> np.ndarray: +def convert_polygons_to_mask(polygons: List, height: int, width: int, value: Optional[int] = 1) -> np.ndarray: """ Converts a list of polygons, encoded as a list of dictionaries into an ``nd.array`` mask. @@ -1386,38 +1297,24 @@ def _parse_version(data: dict) -> dt.AnnotationFileVersion: return dt.AnnotationFileVersion(int(major), int(minor), suffix) -def _data_to_annotations( - data: Dict[str, Any] -) -> List[Union[dt.Annotation, dt.VideoAnnotation]]: +def _data_to_annotations(data: Dict[str, Any]) -> List[Union[dt.Annotation, dt.VideoAnnotation]]: raw_image_annotations = filter( lambda annotation: ( - ("frames" not in annotation) - and ("raster_layer" not in annotation) - and ("mask" not in annotation) + ("frames" not in annotation) and ("raster_layer" not in annotation) and ("mask" not in annotation) ), data["annotations"], ) - raw_video_annotations = filter( - lambda annotation: "frames" in annotation, data["annotations"] - ) - raw_raster_annotations = filter( - lambda annotation: "raster_layer" in annotation, data["annotations"] - ) - raw_mask_annotations = filter( - lambda annotation: "mask" in annotation, data["annotations"] - ) - image_annotations: List[dt.Annotation] = list( - filter(None, map(_parse_darwin_annotation, raw_image_annotations)) - ) + raw_video_annotations = filter(lambda annotation: "frames" in annotation, data["annotations"]) + raw_raster_annotations = filter(lambda annotation: "raster_layer" in annotation, data["annotations"]) + raw_mask_annotations = filter(lambda annotation: "mask" in annotation, data["annotations"]) + image_annotations: List[dt.Annotation] = list(filter(None, map(_parse_darwin_annotation, raw_image_annotations))) video_annotations: List[dt.VideoAnnotation] = list( filter(None, map(_parse_darwin_video_annotation, raw_video_annotations)) ) raster_annotations: List[dt.Annotation] = list( filter(None, map(_parse_darwin_raster_annotation, raw_raster_annotations)) ) - mask_annotations: List[dt.Annotation] = list( - filter(None, map(_parse_darwin_mask_annotation, raw_mask_annotations)) - ) + mask_annotations: List[dt.Annotation] = list(filter(None, map(_parse_darwin_mask_annotation, raw_mask_annotations))) return [ *image_annotations, @@ -1438,6 +1335,4 @@ def _supported_schema_versions() -> Dict[Tuple[int, int, str], str]: def _default_schema(version: dt.AnnotationFileVersion) -> Optional[str]: - return _supported_schema_versions().get( - (version.major, version.minor, version.suffix) - ) + return _supported_schema_versions().get((version.major, version.minor, version.suffix))