From cbcc939df7b6d610fefba94fd57baf2513ebad50 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Wed, 15 Nov 2023 14:55:43 +0000 Subject: [PATCH 1/3] automatic ruff --fix changes --- darwin/cli.py | 3 -- darwin/cli_functions.py | 4 +- darwin/dataset/download_manager.py | 8 ++-- darwin/dataset/remote_dataset.py | 4 +- darwin/dataset/remote_dataset_v1.py | 6 +-- darwin/dataset/remote_dataset_v2.py | 2 +- darwin/dataset/upload_manager.py | 4 +- darwin/datatypes.py | 1 - darwin/exceptions.py | 1 - darwin/exporter/exporter.py | 2 +- darwin/exporter/formats/coco.py | 1 - darwin/exporter/formats/dataloop.py | 1 - darwin/exporter/formats/mask.py | 45 ++++++++----------- darwin/exporter/formats/nifti.py | 8 +--- darwin/exporter/formats/pascalvoc.py | 1 - darwin/exporter/formats/yolo_segmented.py | 5 +-- darwin/importer/formats/coco.py | 6 +-- darwin/importer/formats/csv_tags.py | 8 ++-- darwin/importer/formats/csv_tags_video.py | 8 ++-- darwin/importer/formats/dataloop.py | 7 ++- darwin/importer/formats/labelbox.py | 1 - darwin/importer/formats/nifti.py | 16 +++---- darwin/importer/formats/pascal_voc.py | 6 +-- darwin/importer/importer.py | 4 +- darwin/torch/__init__.py | 2 +- darwin/torch/dataset.py | 2 +- darwin/torch/utils.py | 4 +- darwin/utils/utils.py | 36 +++++++-------- .../importer/formats/import_dataloop_test.py | 11 ++--- 29 files changed, 85 insertions(+), 122 deletions(-) diff --git a/darwin/cli.py b/darwin/cli.py index da18d2ec6..fb78a6502 100644 --- a/darwin/cli.py +++ b/darwin/cli.py @@ -2,10 +2,7 @@ import getpass import os -import platform from argparse import ArgumentParser, Namespace -from datetime import datetime -from json import dumps import requests.exceptions from rich.console import Console diff --git a/darwin/cli_functions.py b/darwin/cli_functions.py index c0ce35322..4caddb77b 100644 --- a/darwin/cli_functions.py +++ b/darwin/cli_functions.py @@ -8,7 +8,7 @@ from glob import glob from itertools import tee from pathlib import Path -from typing import Any, Dict, Iterator, List, NoReturn, Optional, Set, Union +from typing import Dict, Iterator, List, NoReturn, Optional, Set, Union import humanize from rich.console import Console @@ -1065,7 +1065,7 @@ def delete_files( console.print("Cancelled.") return - found_filenames: Set[str] = set([item.filename for item in items_2]) + found_filenames: Set[str] = {item.filename for item in items_2} not_found_filenames: Set[str] = set(files) - found_filenames for filename in not_found_filenames: console.print(f"File not found: {filename}", style="warning") diff --git a/darwin/dataset/download_manager.py b/darwin/dataset/download_manager.py index b851a6c59..4ad1911cc 100644 --- a/darwin/dataset/download_manager.py +++ b/darwin/dataset/download_manager.py @@ -94,7 +94,7 @@ def download_all_images_from_annotations( # Verify that there is not already image in the images folder unfiltered_files = ( - images_path.rglob(f"*") if use_folders else images_path.glob(f"*") + images_path.rglob("*") if use_folders else images_path.glob("*") ) existing_images = { image for image in unfiltered_files if is_image_extension_allowed(image.suffix) @@ -666,9 +666,9 @@ def _extract_frames_from_segment(path: Path, manifest: dt.SegmentManifest) -> No cap = VideoCapture(str(path)) # Read and save frames. Iterates over every frame because frame seeking in OCV is not reliable or guaranteed. - frames_to_extract = dict( - [(item.frame, item.visible_frame) for item in manifest.items if item.visibility] - ) + frames_to_extract = { + item.frame: item.visible_frame for item in manifest.items if item.visibility + } frame_index = 0 while cap.isOpened(): success, frame = cap.read() diff --git a/darwin/dataset/remote_dataset.py b/darwin/dataset/remote_dataset.py index 336523326..c70dab7a1 100644 --- a/darwin/dataset/remote_dataset.py +++ b/darwin/dataset/remote_dataset.py @@ -286,7 +286,7 @@ def pull( continue if video_frames and any( - [not slot.frame_urls for slot in annotation.slots] + not slot.frame_urls for slot in annotation.slots ): # will raise if not installed via pip install darwin-py[ocv] try: @@ -632,7 +632,7 @@ def fetch_remote_classes(self, team_wide=False) -> List[Dict[str, Any]]: classes_to_return = [] for cls in all_classes: belongs_to_current_dataset = any( - [dataset["id"] == self.dataset_id for dataset in cls["datasets"]] + dataset["id"] == self.dataset_id for dataset in cls["datasets"] ) cls["available"] = belongs_to_current_dataset if team_wide or belongs_to_current_dataset: diff --git a/darwin/dataset/remote_dataset_v1.py b/darwin/dataset/remote_dataset_v1.py index 3d9b41ed0..58dde498a 100644 --- a/darwin/dataset/remote_dataset_v1.py +++ b/darwin/dataset/remote_dataset_v1.py @@ -1,6 +1,5 @@ import itertools from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Sequence, Union -from xml.dom import ValidationErr from requests.models import Response @@ -346,9 +345,8 @@ def complete(self, items: Iterator[DatasetItem]) -> None: items : Iterator[DatasetItem] The ``DatasetItem``\\s to be completed. """ - wf_template_id_mapper = lambda item: item.current_workflow[ - "workflow_template_id" - ] + def wf_template_id_mapper(item): + return item.current_workflow["workflow_template_id"] input_items: List[DatasetItem] = list(items) # We split into items with and without workflow diff --git a/darwin/dataset/remote_dataset_v2.py b/darwin/dataset/remote_dataset_v2.py index ecb4817db..70d335d3c 100644 --- a/darwin/dataset/remote_dataset_v2.py +++ b/darwin/dataset/remote_dataset_v2.py @@ -430,7 +430,7 @@ def export( format = "darwin_json_2" elif str_version == "1.0": format = "json" - elif version == None: + elif version is None: format = None else: raise UnknownExportVersion(version) diff --git a/darwin/dataset/upload_manager.py b/darwin/dataset/upload_manager.py index 79b6213b2..58019167e 100644 --- a/darwin/dataset/upload_manager.py +++ b/darwin/dataset/upload_manager.py @@ -76,7 +76,7 @@ def __init__( @staticmethod def parse_v2(payload): if len(payload["slots"]) > 1: - raise NotImplemented("multiple files support not yet implemented") + raise NotImplementedError("multiple files support not yet implemented") slot = payload["slots"][0] return ItemPayload( dataset_item_id=payload.get("id", None), @@ -552,7 +552,7 @@ def upload_function( file_lookup = {file.full_path: file for file in self.local_files} for item in self.pending_items: if len(item.slots) != 1: - raise NotImplemented("Multi file upload is not supported") + raise NotImplementedError("Multi file upload is not supported") upload_id = item.slots[0]["upload_id"] file = file_lookup.get(item.full_path) if not file: diff --git a/darwin/datatypes.py b/darwin/datatypes.py index bd39b90cd..854dfbed8 100644 --- a/darwin/datatypes.py +++ b/darwin/datatypes.py @@ -1,5 +1,4 @@ from dataclasses import dataclass, field -from email.policy import default from enum import Enum, auto from pathlib import Path from typing import ( diff --git a/darwin/exceptions.py b/darwin/exceptions.py index e688dd6ee..e42d3621d 100644 --- a/darwin/exceptions.py +++ b/darwin/exceptions.py @@ -1,4 +1,3 @@ -from cmath import exp from pathlib import Path from pprint import pprint from textwrap import dedent diff --git a/darwin/exporter/exporter.py b/darwin/exporter/exporter.py index 0d1a97ca9..8cf4d95c6 100644 --- a/darwin/exporter/exporter.py +++ b/darwin/exporter/exporter.py @@ -60,7 +60,7 @@ def export_annotations( output_directory : PathLike Where the parsed files will be placed after the operation is complete. """ - print(f"Converting annotations...") + print("Converting annotations...") exporter( darwin_to_dt_gen(file_paths, split_sequences=split_sequences), Path(output_directory), diff --git a/darwin/exporter/formats/coco.py b/darwin/exporter/formats/coco.py index 53e85353b..fbba4d504 100644 --- a/darwin/exporter/formats/coco.py +++ b/darwin/exporter/formats/coco.py @@ -9,7 +9,6 @@ from upolygon import draw_polygon, rle_encode import darwin.datatypes as dt -from darwin.exporter.formats.numpy_encoder import NumpyEncoder from darwin.utils import convert_polygons_to_sequences from darwin.version import __version__ diff --git a/darwin/exporter/formats/dataloop.py b/darwin/exporter/formats/dataloop.py index 1aedd9d77..96fc47bdd 100644 --- a/darwin/exporter/formats/dataloop.py +++ b/darwin/exporter/formats/dataloop.py @@ -5,7 +5,6 @@ import orjson as json import darwin.datatypes as dt -from darwin.exporter.formats.numpy_encoder import NumpyEncoder from darwin.version import __version__ DEPRECATION_MESSAGE = """ diff --git a/darwin/exporter/formats/mask.py b/darwin/exporter/formats/mask.py index 0f7e8b735..071f0b700 100644 --- a/darwin/exporter/formats/mask.py +++ b/darwin/exporter/formats/mask.py @@ -2,9 +2,8 @@ import math import os from csv import writer as csv_writer -from functools import reduce from pathlib import Path -from typing import Dict, Iterable, List, Literal, Optional, Set, Tuple, get_args +from typing import Dict, Iterable, List, Optional, Set, Tuple, get_args import numpy as np @@ -17,7 +16,7 @@ import darwin.datatypes as dt from darwin.exceptions import DarwinException -from darwin.utils import convert_polygons_to_sequences, ispolygon +from darwin.utils import convert_polygons_to_sequences def get_palette(mode: dt.MaskTypes.Mode, categories: List[str]) -> dt.MaskTypes.Palette: @@ -37,7 +36,7 @@ def get_palette(mode: dt.MaskTypes.Mode, categories: List[str]) -> dt.MaskTypes. A dict of categories and their corresponding palette value. """ - if not mode in get_args(dt.MaskTypes.Mode): + if mode not in get_args(dt.MaskTypes.Mode): raise ValueError(f"Unknown mode {mode}.") from DarwinException if not isinstance(categories, list) or not categories: @@ -68,7 +67,7 @@ def get_palette(mode: dt.MaskTypes.Mode, categories: List[str]) -> dt.MaskTypes. if not palette: raise ValueError( - f"Failed to generate a palette.", mode, categories + "Failed to generate a palette.", mode, categories ) from DarwinException return palette @@ -101,14 +100,10 @@ def get_rgb_colours( (x / num_categories, SATURATION_OF_COLOUR, VALUE_OF_COLOUR) for x in range(num_categories - 1) ] - rgb_colour_list: dt.MaskTypes.RgbColorList = list( - map(lambda x: [int(e * 255) for e in colorsys.hsv_to_rgb(*x)], hsv_colours) - ) + rgb_colour_list: dt.MaskTypes.RgbColorList = [[int(e * 255) for e in colorsys.hsv_to_rgb(*x)] for x in hsv_colours] # Now we add BG class with [0 0 0] RGB value rgb_colour_list.insert(0, [0, 0, 0]) - palette_rgb: dt.MaskTypes.RgbPalette = { - c: rgb for c, rgb in zip(categories, rgb_colour_list) - } + palette_rgb: dt.MaskTypes.RgbPalette = dict(zip(categories, rgb_colour_list)) rgb_colours: dt.MaskTypes.RgbColors = [c for e in rgb_colour_list for c in e] return rgb_colours, palette_rgb @@ -195,7 +190,7 @@ def colours_in_rle( f"Could not find mask with uuid {uuid} in mask lookup table." ) - if not mask.name in colours: + if mask.name not in colours: colours[mask.name] = colour_value return colours # Returns same item as the outset, technically not needed, but best practice. @@ -235,7 +230,7 @@ def get_or_generate_colour(cat_name: str, colours: dt.MaskTypes.ColoursDict) -> ------- int - the integer for the colour name. These will later be reassigned to a wider spread across the colour spectrum. """ - if not cat_name in colours: + if cat_name not in colours: colours[cat_name] = len(colours) + 1 return colours[cat_name] @@ -293,7 +288,7 @@ def render_polygons( for a in filtered_annotations: try: cat = a.annotation_class.name - if not cat in categories: + if cat not in categories: categories.append(cat) if a.annotation_class.annotation_type == "polygon": @@ -368,7 +363,7 @@ def render_raster( mask_annotations: List[dt.AnnotationMask] = [] raster_layer: Optional[dt.RasterLayer] = None - mask_lookup: Dict[str, dt.AnnotationMask] = dict() + mask_lookup: Dict[str, dt.AnnotationMask] = {} for a in annotations: if isinstance(a, dt.VideoAnnotation): @@ -390,11 +385,11 @@ def render_raster( mask_annotations.append(new_mask) - if not new_mask.id in mask_lookup: + if new_mask.id not in mask_lookup: mask_lookup[new_mask.id] = new_mask # Add the category to the list of categories - if not new_mask.name in categories: + if new_mask.name not in categories: categories.append(new_mask.name) if a.annotation_class.annotation_type == "raster_layer" and (rl := data): @@ -415,11 +410,11 @@ def render_raster( raster_layer = new_rl if not raster_layer: - errors.append(ValueError(f"Annotation has no raster layer")) + errors.append(ValueError("Annotation has no raster layer")) return errors, mask, categories, colours if not mask_annotations: - errors.append(ValueError(f"Annotation has no masks")) + errors.append(ValueError("Annotation has no masks")) return errors, mask, categories, colours try: @@ -447,19 +442,17 @@ def export( if len(all_classes_sets) > 0: all_classes: Set[dt.AnnotationClass] = set.union(*all_classes_sets) categories: List[str] = ["__background__"] + sorted( - list( - set( - [c.name for c in all_classes if c.annotation_type in accepted_types] - ) - ), + { + c.name for c in all_classes if c.annotation_type in accepted_types + }, key=lambda x: x.lower(), ) palette = get_palette(mode, categories) else: categories = ["__background__"] - palette = dict() + palette = {} - colours: dt.MaskTypes.ColoursDict = dict() + colours: dt.MaskTypes.ColoursDict = {} for annotation_file in annotation_files: image_rel_path = os.path.splitext(annotation_file.full_path)[0].lstrip("/") diff --git a/darwin/exporter/formats/nifti.py b/darwin/exporter/formats/nifti.py index dacf21b7a..527c70846 100644 --- a/darwin/exporter/formats/nifti.py +++ b/darwin/exporter/formats/nifti.py @@ -1,6 +1,5 @@ import ast import json as native_json -from asyncore import loop from dataclasses import dataclass from pathlib import Path from typing import Dict, Iterable, List, Optional, Tuple, Union @@ -19,11 +18,9 @@ console.print(import_fail_string) exit() import numpy as np -import orjson as json -from PIL import Image import darwin.datatypes as dt -from darwin.utils import convert_polygons_to_mask, get_progress_bar +from darwin.utils import convert_polygons_to_mask @dataclass @@ -141,7 +138,6 @@ def check_for_error_and_return_imageid( """ - output_volumes = None filename = Path(video_annotation.filename) try: suffixes = filename.suffixes[-2:] @@ -277,7 +273,7 @@ def populate_output_volumes( ) else: continue - class_name = frames[frame_idx].annotation_class.name + frames[frame_idx].annotation_class.name im_mask = convert_polygons_to_mask(polygons, height=height, width=width) volume = output_volumes[series_instance_uid] if view_idx == 0: diff --git a/darwin/exporter/formats/pascalvoc.py b/darwin/exporter/formats/pascalvoc.py index af8acffa4..aa7acd08b 100644 --- a/darwin/exporter/formats/pascalvoc.py +++ b/darwin/exporter/formats/pascalvoc.py @@ -3,7 +3,6 @@ from xml.etree.ElementTree import Element, SubElement, tostring import deprecation -import orjson as json import darwin.datatypes as dt from darwin.utils import attempt_decode diff --git a/darwin/exporter/formats/yolo_segmented.py b/darwin/exporter/formats/yolo_segmented.py index ef599e7e5..88702c1c7 100644 --- a/darwin/exporter/formats/yolo_segmented.py +++ b/darwin/exporter/formats/yolo_segmented.py @@ -1,11 +1,10 @@ from collections import namedtuple from enum import Enum, auto from logging import getLogger -from multiprocessing.pool import CLOSE from pathlib import Path from typing import Iterable, List -from darwin.datatypes import Annotation, AnnotationFile, JSONType, VideoAnnotation +from darwin.datatypes import Annotation, AnnotationFile, VideoAnnotation from darwin.exceptions import DarwinException from darwin.exporter.formats.helpers.yolo_class_builder import ( ClassIndex, @@ -208,7 +207,7 @@ def _handle_polygon( ) return False - except Exception as exc: + except Exception: logger.error( f"An unexpected error occured while exporting annotation at index {annotation_index}." ) diff --git a/darwin/importer/formats/coco.py b/darwin/importer/formats/coco.py index 05034b479..b81fb8a4d 100644 --- a/darwin/importer/formats/coco.py +++ b/darwin/importer/formats/coco.py @@ -97,9 +97,9 @@ def parse_json( for image_id in image_annotations.keys(): image = image_lookup_table[int(image_id)] annotations = list(filter(None, image_annotations[image_id])) - annotation_classes = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes = { + annotation.annotation_class for annotation in annotations + } remote_path, filename = deconstruct_full_path(image["file_name"]) yield dt.AnnotationFile( path, filename, annotation_classes, annotations, remote_path=remote_path diff --git a/darwin/importer/formats/csv_tags.py b/darwin/importer/formats/csv_tags.py index f181b8296..f10a87880 100644 --- a/darwin/importer/formats/csv_tags.py +++ b/darwin/importer/formats/csv_tags.py @@ -28,13 +28,13 @@ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]: with path.open() as f: reader = csv.reader(f) for row in reader: - filename, *tags = map(lambda s: s.strip(), row) + filename, *tags = (s.strip() for s in row) if filename == "": continue annotations = [dt.make_tag(tag) for tag in tags if len(tag) > 0] - annotation_classes = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes = { + annotation.annotation_class for annotation in annotations + } remote_path, filename = deconstruct_full_path(filename) files.append( dt.AnnotationFile( diff --git a/darwin/importer/formats/csv_tags_video.py b/darwin/importer/formats/csv_tags_video.py index 35f05e230..2d35abe68 100644 --- a/darwin/importer/formats/csv_tags_video.py +++ b/darwin/importer/formats/csv_tags_video.py @@ -30,7 +30,7 @@ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]: reader = csv.reader(f) for row in reader: try: - filename, tag, start_frame, end_frame = map(lambda s: s.strip(), row) + filename, tag, start_frame, end_frame = (s.strip() for s in row) except ValueError: continue if filename == "": @@ -51,9 +51,9 @@ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]: file_annotation_map[filename].append(annotation) for filename in file_annotation_map: annotations = file_annotation_map[filename] - annotation_classes = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes = { + annotation.annotation_class for annotation in annotations + } files.append( dt.AnnotationFile( path, diff --git a/darwin/importer/formats/dataloop.py b/darwin/importer/formats/dataloop.py index 207027316..4f546ab09 100644 --- a/darwin/importer/formats/dataloop.py +++ b/darwin/importer/formats/dataloop.py @@ -1,7 +1,6 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Set -import orjson as json import darwin.datatypes as dt from darwin.exceptions import ( @@ -33,9 +32,9 @@ def parse_path(path: Path) -> Optional[dt.AnnotationFile]: annotations: List[dt.Annotation] = list( filter(None, map(_parse_annotation, data["annotations"])) ) - annotation_classes: Set[dt.AnnotationClass] = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes: Set[dt.AnnotationClass] = { + annotation.annotation_class for annotation in annotations + } return dt.AnnotationFile( path, _remove_leading_slash(data["filename"]), diff --git a/darwin/importer/formats/labelbox.py b/darwin/importer/formats/labelbox.py index d6cd48eaa..d8b66fb46 100644 --- a/darwin/importer/formats/labelbox.py +++ b/darwin/importer/formats/labelbox.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Set, cast -import orjson as json from jsonschema import validate from darwin.datatypes import ( diff --git a/darwin/importer/formats/nifti.py b/darwin/importer/formats/nifti.py index 49be7b951..257bd9542 100644 --- a/darwin/importer/formats/nifti.py +++ b/darwin/importer/formats/nifti.py @@ -1,11 +1,9 @@ import sys import warnings -import zipfile from collections import OrderedDict, defaultdict from pathlib import Path from typing import Dict, List, Optional, Sequence, Tuple, Union -import orjson as json from rich.console import Console from darwin.utils import attempt_decode @@ -27,7 +25,6 @@ import darwin.datatypes as dt from darwin.importer.formats.nifti_schemas import nifti_import_schema -from darwin.version import __version__ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]: @@ -56,7 +53,7 @@ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]: data = attempt_decode(path) try: validate(data, schema=nifti_import_schema) - except Exception as e: + except Exception: console.print( "Skipping file: {} (invalid json file, see schema for details)".format( path @@ -96,7 +93,6 @@ def _parse_nifti( ) -> dt.AnnotationFile: img, pixdims = process_nifti(nib.load(nifti_path)) - shape = img.shape processed_class_map = process_class_map(class_map) video_annotations = [] if mode == "instances": # For each instance produce a video annotation @@ -131,12 +127,10 @@ def _parse_nifti( if _video_annotations is None: continue video_annotations += _video_annotations - annotation_classes = set( - [ - dt.AnnotationClass(class_name, "polygon", "polygon") + annotation_classes = { + dt.AnnotationClass(class_name, "polygon", "polygon") for class_name in class_map.values() - ] - ) + } return dt.AnnotationFile( path=json_path, filename=str(filename), @@ -376,7 +370,7 @@ def process_nifti( if isinstance(input_data, nib.nifti1.Nifti1Image): img = correct_nifti_header_if_necessary(input_data) img = nib.funcs.as_closest_canonical(img) - axcodes = nib.orientations.aff2axcodes(img.affine) + nib.orientations.aff2axcodes(img.affine) # TODO: Future feature to pass custom ornt could go here. ornt = [[0.0, -1.0], [1.0, -1.0], [1.0, -1.0]] data_array = nib.orientations.apply_orientation(img.get_fdata(), ornt) diff --git a/darwin/importer/formats/pascal_voc.py b/darwin/importer/formats/pascal_voc.py index f8abf441d..65ed85aba 100644 --- a/darwin/importer/formats/pascal_voc.py +++ b/darwin/importer/formats/pascal_voc.py @@ -55,9 +55,9 @@ def parse_path(path: Path) -> Optional[dt.AnnotationFile]: annotations: List[dt.Annotation] = list( filter(None, map(_parse_annotation, root.findall("object"))) ) - annotation_classes = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes = { + annotation.annotation_class for annotation in annotations + } return dt.AnnotationFile( path, filename, annotation_classes, annotations, remote_path="/" diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index 06b5f5af7..d2f02f807 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -603,9 +603,7 @@ def _warn_unsupported_annotations(parsed_files: List[AnnotationFile]) -> None: if annotation.annotation_class.annotation_type in UNSUPPORTED_CLASSES: skipped_annotations.append(annotation) if len(skipped_annotations) > 0: - types = set( - map(lambda c: c.annotation_class.annotation_type, skipped_annotations) - ) # noqa: C417 + types = {c.annotation_class.annotation_type for c in skipped_annotations} # noqa: C417 console.print( f"Import of annotation class types '{', '.join(types)}' is not yet supported. Skipping {len(skipped_annotations)} " + "annotations from '{parsed_file.full_path}'.\n", diff --git a/darwin/torch/__init__.py b/darwin/torch/__init__.py index ef55392ff..4c4fca583 100644 --- a/darwin/torch/__init__.py +++ b/darwin/torch/__init__.py @@ -15,7 +15,7 @@ import torch # noqa except ImportError: raise ImportError( - f"darwin.torch requires pytorch and torchvision. Install it using: pip install torch torchvision" + "darwin.torch requires pytorch and torchvision. Install it using: pip install torch torchvision" ) from None from .dataset import get_dataset # noqa diff --git a/darwin/torch/dataset.py b/darwin/torch/dataset.py index 080fef467..c324e36f3 100644 --- a/darwin/torch/dataset.py +++ b/darwin/torch/dataset.py @@ -421,7 +421,7 @@ def __init__( self, transform: Optional[Union[List[Callable], Callable]] = None, **kwargs ): super().__init__(annotation_type="polygon", **kwargs) - if not "__background__" in self.classes: + if "__background__" not in self.classes: self.classes.insert(0, "__background__") self.num_classes += 1 if transform is not None and isinstance(transform, list): diff --git a/darwin/torch/utils.py b/darwin/torch/utils.py index f5ffcac94..00e20178b 100644 --- a/darwin/torch/utils.py +++ b/darwin/torch/utils.py @@ -1,7 +1,7 @@ import os import sys from pathlib import Path -from typing import Iterable, List, Optional, Tuple, Union +from typing import Iterable, List, Optional, Tuple import numpy as np import torch @@ -31,7 +31,7 @@ def flatten_masks_by_category(masks: torch.Tensor, cats: List[int]) -> torch.Ten assert isinstance(masks, torch.Tensor) assert isinstance(cats, List) assert masks.shape[0] == len(cats) - order_of_polygons = [i for i in range(1, len(cats) + 1)] + order_of_polygons = list(range(1, len(cats) + 1)) polygon_mapping = {order: cat for cat, order in zip(cats, order_of_polygons)} BACKGROUND: int = 0 polygon_mapping[BACKGROUND] = 0 diff --git a/darwin/utils/utils.py b/darwin/utils/utils.py index 1d4f135af..10255a8c0 100644 --- a/darwin/utils/utils.py +++ b/darwin/utils/utils.py @@ -25,18 +25,16 @@ import orjson as json import requests from json_stream.base import PersistentStreamingJSONObject -from jsonschema import exceptions, validators -from requests import Response, request +from jsonschema import validators +from requests import Response from rich.progress import ProgressType, track from upolygon import draw_polygon import darwin.datatypes as dt from darwin.config import Config from darwin.exceptions import ( - AnnotationFileValidationError, MissingSchema, OutdatedDarwinJSONFormat, - UnknownAnnotationFileSchema, UnrecognizableFileEncoding, UnsupportedFileType, ) @@ -89,7 +87,7 @@ def is_extension_allowed_by_filename(filename: str) -> bool: bool Whether or not the given extension of the filename is allowed. """ - return any([filename.lower().endswith(ext) for ext in SUPPORTED_EXTENSIONS]) + return any(filename.lower().endswith(ext) for ext in SUPPORTED_EXTENSIONS) @deprecation.deprecated(deprecated_in="0.8.4", current_version=__version__) @@ -126,7 +124,7 @@ def is_image_extension_allowed_by_filename(filename: str) -> bool: bool Whether or not the given extension is allowed. """ - return any([filename.lower().endswith(ext) for ext in SUPPORTED_IMAGE_EXTENSIONS]) + return any(filename.lower().endswith(ext) for ext in SUPPORTED_IMAGE_EXTENSIONS) @deprecation.deprecated(deprecated_in="0.8.4", current_version=__version__) @@ -161,7 +159,7 @@ def is_video_extension_allowed_by_filename(extension: str) -> bool: bool Whether or not the given extension is allowed. """ - return any([extension.lower().endswith(ext) for ext in SUPPORTED_VIDEO_EXTENSIONS]) + return any(extension.lower().endswith(ext) for ext in SUPPORTED_VIDEO_EXTENSIONS) @deprecation.deprecated(deprecated_in="0.8.4", current_version=__version__) @@ -535,9 +533,9 @@ def _parse_darwin_v2(path: Path, data: Dict[str, Any]) -> dt.AnnotationFile: annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations( data ) - annotation_classes: Set[dt.AnnotationClass] = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes: Set[dt.AnnotationClass] = { + annotation.annotation_class for annotation in annotations + } if len(slots) == 0: annotation_file = dt.AnnotationFile( @@ -613,9 +611,9 @@ def _parse_darwin_image( annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations( data ) - annotation_classes: Set[dt.AnnotationClass] = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes: Set[dt.AnnotationClass] = { + annotation.annotation_class for annotation in annotations + } slot = dt.Slot( name=None, @@ -658,9 +656,9 @@ def _parse_darwin_video( annotations: List[Union[dt.Annotation, dt.VideoAnnotation]] = _data_to_annotations( data ) - annotation_classes: Set[dt.AnnotationClass] = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes: Set[dt.AnnotationClass] = { + annotation.annotation_class for annotation in annotations + } if "width" not in data["image"] or "height" not in data["image"]: raise OutdatedDarwinJSONFormat( @@ -1005,9 +1003,9 @@ def split_video_annotation(annotation: dt.AnnotationFile) -> List[dt.AnnotationF for a in annotation.annotations if isinstance(a, dt.VideoAnnotation) and i in a.frames ] - annotation_classes: Set[dt.AnnotationClass] = set( - [annotation.annotation_class for annotation in annotations] - ) + annotation_classes: Set[dt.AnnotationClass] = { + annotation.annotation_class for annotation in annotations + } filename: str = f"{Path(annotation.filename).stem}/{i:07d}.png" frame_annotations.append( dt.AnnotationFile( diff --git a/tests/darwin/importer/formats/import_dataloop_test.py b/tests/darwin/importer/formats/import_dataloop_test.py index 944d6be57..de4e4059e 100644 --- a/tests/darwin/importer/formats/import_dataloop_test.py +++ b/tests/darwin/importer/formats/import_dataloop_test.py @@ -58,24 +58,21 @@ def test_returns_none_if_file_extension_is_not_json( @patch( "darwin.importer.formats.dataloop._remove_leading_slash", ) - @patch("darwin.importer.formats.dataloop.json.loads") - @patch("darwin.importer.formats.dataloop.Path.open") + @patch("darwin.importer.formats.dataloop.attempt_decode") @patch("darwin.importer.formats.dataloop._parse_annotation") def test_opens_annotations_file_and_parses( self, _parse_annotation_mock: MagicMock, - path_open_mock: MagicMock, - json_load_mock: MagicMock, + attempt_decode_mock: MagicMock, mock_remove_leading_slash: MagicMock, ): - json_load_mock.return_value = self.DARWIN_PARSED_DATA + attempt_decode_mock.return_value = self.DARWIN_PARSED_DATA test_path = "foo.json" parse_path(Path(test_path)) self.assertEqual(_parse_annotation_mock.call_count, 3) - path_open_mock.assert_called_once() - json_load_mock.assert_called_once() + attempt_decode_mock.assert_called_once() mock_remove_leading_slash.assert_called_once() From 4029bf317e8b2e473aa6e2d7b03521543ea3e926 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Wed, 15 Nov 2023 15:07:36 +0000 Subject: [PATCH 2/3] black changes --- darwin/backend_v2.py | 3 ++- darwin/dataset/download_manager.py | 4 +--- darwin/dataset/remote_dataset_v1.py | 2 ++ darwin/exporter/formats/mask.py | 8 ++++---- darwin/importer/formats/coco.py | 4 +--- darwin/importer/formats/csv_tags_video.py | 4 +--- darwin/importer/formats/nifti.py | 2 +- darwin/importer/formats/pascal_voc.py | 4 +--- darwin/importer/importer.py | 4 +++- 9 files changed, 16 insertions(+), 19 deletions(-) diff --git a/darwin/backend_v2.py b/darwin/backend_v2.py index dca5327f6..fd0b67741 100644 --- a/darwin/backend_v2.py +++ b/darwin/backend_v2.py @@ -1,6 +1,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from urllib import parse +from darwin.client import Client from darwin.datatypes import ItemId @@ -18,7 +19,7 @@ def wrapper(self, *args, **kwargs) -> Callable: class BackendV2: - def __init__(self, client: "Client", default_team): + def __init__(self, client: Client, default_team): self._client = client self._default_team = default_team diff --git a/darwin/dataset/download_manager.py b/darwin/dataset/download_manager.py index 4ad1911cc..de99672d3 100644 --- a/darwin/dataset/download_manager.py +++ b/darwin/dataset/download_manager.py @@ -93,9 +93,7 @@ def download_all_images_from_annotations( raise ValueError(f"Annotation format {annotation_format} not supported") # Verify that there is not already image in the images folder - unfiltered_files = ( - images_path.rglob("*") if use_folders else images_path.glob("*") - ) + unfiltered_files = images_path.rglob("*") if use_folders else images_path.glob("*") existing_images = { image for image in unfiltered_files if is_image_extension_allowed(image.suffix) } diff --git a/darwin/dataset/remote_dataset_v1.py b/darwin/dataset/remote_dataset_v1.py index 58dde498a..f5a2a36e7 100644 --- a/darwin/dataset/remote_dataset_v1.py +++ b/darwin/dataset/remote_dataset_v1.py @@ -345,8 +345,10 @@ def complete(self, items: Iterator[DatasetItem]) -> None: items : Iterator[DatasetItem] The ``DatasetItem``\\s to be completed. """ + def wf_template_id_mapper(item): return item.current_workflow["workflow_template_id"] + input_items: List[DatasetItem] = list(items) # We split into items with and without workflow diff --git a/darwin/exporter/formats/mask.py b/darwin/exporter/formats/mask.py index 071f0b700..78e7037db 100644 --- a/darwin/exporter/formats/mask.py +++ b/darwin/exporter/formats/mask.py @@ -100,7 +100,9 @@ def get_rgb_colours( (x / num_categories, SATURATION_OF_COLOUR, VALUE_OF_COLOUR) for x in range(num_categories - 1) ] - rgb_colour_list: dt.MaskTypes.RgbColorList = [[int(e * 255) for e in colorsys.hsv_to_rgb(*x)] for x in hsv_colours] + rgb_colour_list: dt.MaskTypes.RgbColorList = [ + [int(e * 255) for e in colorsys.hsv_to_rgb(*x)] for x in hsv_colours + ] # Now we add BG class with [0 0 0] RGB value rgb_colour_list.insert(0, [0, 0, 0]) palette_rgb: dt.MaskTypes.RgbPalette = dict(zip(categories, rgb_colour_list)) @@ -442,9 +444,7 @@ def export( if len(all_classes_sets) > 0: all_classes: Set[dt.AnnotationClass] = set.union(*all_classes_sets) categories: List[str] = ["__background__"] + sorted( - { - c.name for c in all_classes if c.annotation_type in accepted_types - }, + {c.name for c in all_classes if c.annotation_type in accepted_types}, key=lambda x: x.lower(), ) palette = get_palette(mode, categories) diff --git a/darwin/importer/formats/coco.py b/darwin/importer/formats/coco.py index b81fb8a4d..28d4f4f8f 100644 --- a/darwin/importer/formats/coco.py +++ b/darwin/importer/formats/coco.py @@ -97,9 +97,7 @@ def parse_json( for image_id in image_annotations.keys(): image = image_lookup_table[int(image_id)] annotations = list(filter(None, image_annotations[image_id])) - annotation_classes = { - annotation.annotation_class for annotation in annotations - } + annotation_classes = {annotation.annotation_class for annotation in annotations} remote_path, filename = deconstruct_full_path(image["file_name"]) yield dt.AnnotationFile( path, filename, annotation_classes, annotations, remote_path=remote_path diff --git a/darwin/importer/formats/csv_tags_video.py b/darwin/importer/formats/csv_tags_video.py index 2d35abe68..00e9a0c37 100644 --- a/darwin/importer/formats/csv_tags_video.py +++ b/darwin/importer/formats/csv_tags_video.py @@ -51,9 +51,7 @@ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]: file_annotation_map[filename].append(annotation) for filename in file_annotation_map: annotations = file_annotation_map[filename] - annotation_classes = { - annotation.annotation_class for annotation in annotations - } + annotation_classes = {annotation.annotation_class for annotation in annotations} files.append( dt.AnnotationFile( path, diff --git a/darwin/importer/formats/nifti.py b/darwin/importer/formats/nifti.py index 257bd9542..9254a6821 100644 --- a/darwin/importer/formats/nifti.py +++ b/darwin/importer/formats/nifti.py @@ -129,7 +129,7 @@ def _parse_nifti( video_annotations += _video_annotations annotation_classes = { dt.AnnotationClass(class_name, "polygon", "polygon") - for class_name in class_map.values() + for class_name in class_map.values() } return dt.AnnotationFile( path=json_path, diff --git a/darwin/importer/formats/pascal_voc.py b/darwin/importer/formats/pascal_voc.py index 65ed85aba..4470a6149 100644 --- a/darwin/importer/formats/pascal_voc.py +++ b/darwin/importer/formats/pascal_voc.py @@ -55,9 +55,7 @@ def parse_path(path: Path) -> Optional[dt.AnnotationFile]: annotations: List[dt.Annotation] = list( filter(None, map(_parse_annotation, root.findall("object"))) ) - annotation_classes = { - annotation.annotation_class for annotation in annotations - } + annotation_classes = {annotation.annotation_class for annotation in annotations} return dt.AnnotationFile( path, filename, annotation_classes, annotations, remote_path="/" diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index d2f02f807..7bc12edc6 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -603,7 +603,9 @@ def _warn_unsupported_annotations(parsed_files: List[AnnotationFile]) -> None: if annotation.annotation_class.annotation_type in UNSUPPORTED_CLASSES: skipped_annotations.append(annotation) if len(skipped_annotations) > 0: - types = {c.annotation_class.annotation_type for c in skipped_annotations} # noqa: C417 + types = { + c.annotation_class.annotation_type for c in skipped_annotations + } # noqa: C417 console.print( f"Import of annotation class types '{', '.join(types)}' is not yet supported. Skipping {len(skipped_annotations)} " + "annotations from '{parsed_file.full_path}'.\n", From 08ccdc4cc38dd99803b0ff424acf378acdcd72c1 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Wed, 15 Nov 2023 15:08:27 +0000 Subject: [PATCH 3/3] revert of client --- darwin/backend_v2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/darwin/backend_v2.py b/darwin/backend_v2.py index fd0b67741..dca5327f6 100644 --- a/darwin/backend_v2.py +++ b/darwin/backend_v2.py @@ -1,7 +1,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from urllib import parse -from darwin.client import Client from darwin.datatypes import ItemId @@ -19,7 +18,7 @@ def wrapper(self, *args, **kwargs) -> Callable: class BackendV2: - def __init__(self, client: Client, default_team): + def __init__(self, client: "Client", default_team): self._client = client self._default_team = default_team