From eccb7f1fdf991aa1c19400e917d65f508f1b2ba5 Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Tue, 30 May 2023 16:14:58 +0900 Subject: [PATCH 01/10] Change Environment() to DEFAULT_ENVIRONMENT Signed-off-by: Kim, Vinnam --- datumaro/cli/commands/convert.py | 10 ++++++---- datumaro/cli/commands/detect_format.py | 4 ++-- datumaro/cli/commands/download.py | 6 +++--- datumaro/cli/commands/merge.py | 4 ++-- datumaro/cli/commands/patch.py | 4 ++-- .../cli/commands/require_project/modification/add.py | 6 +++--- .../commands/require_project/modification/export.py | 6 +++--- .../commands/require_project/modification/import_.py | 6 +++--- datumaro/cli/commands/transform.py | 6 +++--- datumaro/cli/commands/validate.py | 4 ++-- 10 files changed, 29 insertions(+), 27 deletions(-) diff --git a/datumaro/cli/commands/convert.py b/datumaro/cli/commands/convert.py index 172e200759..eec361ecd7 100644 --- a/datumaro/cli/commands/convert.py +++ b/datumaro/cli/commands/convert.py @@ -8,7 +8,7 @@ import os.path as osp from datumaro.components.dataset import Dataset -from datumaro.components.project import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.util.os_util import make_file_name from ..util import MultilineFormatter @@ -17,8 +17,10 @@ def build_parser(parser_ctor=argparse.ArgumentParser): - builtin_readers = sorted(set(Environment().importers) | set(Environment().extractors)) - builtin_writers = sorted(Environment().exporters) + builtin_readers = sorted( + set(DEFAULT_ENVIRONMENT.importers) | set(DEFAULT_ENVIRONMENT.extractors) + ) + builtin_writers = sorted(DEFAULT_ENVIRONMENT.exporters) parser = parser_ctor( help="Convert an existing dataset to another format", @@ -98,7 +100,7 @@ def get_sensitive_args(): def convert_command(args): - env = Environment() + env = DEFAULT_ENVIRONMENT try: exporter = env.exporters[args.output_format] diff --git a/datumaro/cli/commands/detect_format.py b/datumaro/cli/commands/detect_format.py index 4251e43270..22ef07c219 100644 --- a/datumaro/cli/commands/detect_format.py +++ b/datumaro/cli/commands/detect_format.py @@ -6,7 +6,7 @@ from datumaro.cli.util import MultilineFormatter from datumaro.cli.util.project import load_project -from datumaro.components.environment import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.errors import ProjectNotFoundError from datumaro.components.format_detection import RejectionReason from datumaro.util import dump_json_file @@ -77,7 +77,7 @@ def detect_format_command(args): if project is not None: env = project.env else: - env = Environment() + env = DEFAULT_ENVIRONMENT report = {"rejected_formats": {}} diff --git a/datumaro/cli/commands/download.py b/datumaro/cli/commands/download.py index e67e818056..f3be58ce68 100644 --- a/datumaro/cli/commands/download.py +++ b/datumaro/cli/commands/download.py @@ -10,12 +10,12 @@ import sys from typing import Dict +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.extractor_tfds import ( AVAILABLE_TFDS_DATASETS, TFDS_EXTRACTOR_AVAILABLE, TfdsDatasetRemoteMetadata, ) -from datumaro.components.project import Environment from datumaro.util import dump_json from datumaro.util.os_util import make_file_name @@ -48,7 +48,7 @@ def build_parser(parser_ctor=argparse.ArgumentParser): def build_get_subparser(subparsers: argparse._SubParsersAction): - builtin_writers = sorted(Environment().exporters) + builtin_writers = sorted(DEFAULT_ENVIRONMENT.exporters) if TFDS_EXTRACTOR_AVAILABLE: available_datasets = ", ".join(f"tfds:{name}" for name in AVAILABLE_TFDS_DATASETS) else: @@ -137,7 +137,7 @@ def get_sensitive_args(): def download_command(args): - env = Environment() + env = DEFAULT_ENVIRONMENT if args.dataset_id.startswith("tfds:"): if TFDS_EXTRACTOR_AVAILABLE: diff --git a/datumaro/cli/commands/merge.py b/datumaro/cli/commands/merge.py index db9d9aaa92..52ce33b3bc 100644 --- a/datumaro/cli/commands/merge.py +++ b/datumaro/cli/commands/merge.py @@ -8,7 +8,7 @@ import os.path as osp from datumaro.components.dataset import DEFAULT_FORMAT -from datumaro.components.environment import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.errors import ProjectNotFoundError from datumaro.components.hl_ops import HLOps from datumaro.components.merge.intersect_merge import IntersectMerge @@ -232,7 +232,7 @@ def merge_command(args): if project is not None: env = project.env else: - env = Environment() + env = DEFAULT_ENVIRONMENT try: exporter = env.exporters[args.format] diff --git a/datumaro/cli/commands/patch.py b/datumaro/cli/commands/patch.py index 88164b6c00..bde88aa3d4 100644 --- a/datumaro/cli/commands/patch.py +++ b/datumaro/cli/commands/patch.py @@ -7,7 +7,7 @@ import os import os.path as osp -from datumaro.components.environment import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.errors import ProjectNotFoundError from datumaro.util.scope import scope_add, scoped @@ -131,7 +131,7 @@ def patch_command(args): if project is not None: env = project.env else: - env = Environment() + env = DEFAULT_ENVIRONMENT target_dataset, _project = parse_full_revpath(args.target, project) if _project is not None: diff --git a/datumaro/cli/commands/require_project/modification/add.py b/datumaro/cli/commands/require_project/modification/add.py index db048e714e..71f9174ae9 100644 --- a/datumaro/cli/commands/require_project/modification/add.py +++ b/datumaro/cli/commands/require_project/modification/add.py @@ -6,7 +6,7 @@ import argparse import logging as log -from datumaro.components.environment import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.errors import ProjectNotFoundError from datumaro.util.scope import on_error_do, scope_add, scoped @@ -21,7 +21,7 @@ def build_parser(parser_ctor=argparse.ArgumentParser): - env = Environment() + env = DEFAULT_ENVIRONMENT builtins = sorted(set(env.extractors) | set(env.importers)) parser = parser_ctor( @@ -125,7 +125,7 @@ def add_command(args): if project is not None: env = project.env else: - env = Environment() + env = DEFAULT_ENVIRONMENT fmt = args.format if fmt in env.importers: diff --git a/datumaro/cli/commands/require_project/modification/export.py b/datumaro/cli/commands/require_project/modification/export.py index 3eefa3bc8a..4a11be8410 100644 --- a/datumaro/cli/commands/require_project/modification/export.py +++ b/datumaro/cli/commands/require_project/modification/export.py @@ -7,7 +7,7 @@ import os import os.path as osp -from datumaro.components.environment import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.errors import ProjectNotFoundError from datumaro.components.project import ProjectBuildTargets from datumaro.util.os_util import make_file_name @@ -24,7 +24,7 @@ def build_parser(parser_ctor=argparse.ArgumentParser): - builtins = sorted(Environment().exporters) + builtins = sorted(DEFAULT_ENVIRONMENT.exporters) parser = parser_ctor( help="Export project", @@ -143,7 +143,7 @@ def export_command(args): if project is not None: env = project.env else: - env = Environment() + env = DEFAULT_ENVIRONMENT try: exporter = env.exporters[args.format] diff --git a/datumaro/cli/commands/require_project/modification/import_.py b/datumaro/cli/commands/require_project/modification/import_.py index 2cfaeff956..9774bdd52b 100644 --- a/datumaro/cli/commands/require_project/modification/import_.py +++ b/datumaro/cli/commands/require_project/modification/import_.py @@ -6,7 +6,7 @@ import logging as log import os -from datumaro.components.environment import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.errors import ProjectNotFoundError from datumaro.util.scope import on_error_do, scope_add, scoped @@ -21,7 +21,7 @@ def build_parser(parser_ctor=argparse.ArgumentParser): - env = Environment() + env = DEFAULT_ENVIRONMENT builtins = sorted(set(env.extractors) | set(env.importers)) parser = parser_ctor( @@ -131,7 +131,7 @@ def import_command(args): if project is not None: env = project.env else: - env = Environment() + env = DEFAULT_ENVIRONMENT fmt = args.format if fmt in env.importers: diff --git a/datumaro/cli/commands/transform.py b/datumaro/cli/commands/transform.py index 45fda030b3..12def4d0d1 100644 --- a/datumaro/cli/commands/transform.py +++ b/datumaro/cli/commands/transform.py @@ -7,7 +7,7 @@ import os import os.path as osp -from datumaro.components.environment import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.errors import ProjectNotFoundError from datumaro.components.project import ProjectBuildTargets from datumaro.util import str_to_bool @@ -19,7 +19,7 @@ def build_parser(parser_ctor=argparse.ArgumentParser): - builtins = sorted(Environment().transforms) + builtins = sorted(DEFAULT_ENVIRONMENT.transforms) parser = parser_ctor( help="Transform project", @@ -166,7 +166,7 @@ def transform_command(args): if project is not None: env = project.env else: - env = Environment() + env = DEFAULT_ENVIRONMENT try: transform = env.transforms[args.transform] diff --git a/datumaro/cli/commands/validate.py b/datumaro/cli/commands/validate.py index 3fefc6fe51..21d509da94 100644 --- a/datumaro/cli/commands/validate.py +++ b/datumaro/cli/commands/validate.py @@ -5,7 +5,7 @@ import argparse import logging as log -from datumaro.components.environment import Environment +from datumaro.components.environment import DEFAULT_ENVIRONMENT from datumaro.components.errors import ProjectNotFoundError from datumaro.components.validator import TaskType from datumaro.util import dump_json_file @@ -113,7 +113,7 @@ def validate_command(args): if project is not None: env = project.env else: - env = Environment() + env = DEFAULT_ENVIRONMENT try: validator_type = env.validators[args.task] From cd39895c7657dbf68e76489749afc908177425ad Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Tue, 30 May 2023 16:09:43 +0900 Subject: [PATCH 02/10] Lazy import plugins Signed-off-by: Kim, Vinnam --- datumaro/components/dataset.py | 3 +- datumaro/components/environment.py | 58 ++++++++++++++++++++++++----- datumaro/components/lazy_plugin.py | 59 ++++++++++++++++++++++++++++++ datumaro/plugins/specs.json | 1 + datumaro/plugins/specs.py | 49 +++++++++++++++++++++++++ tests/unit/test_dataset.py | 52 +++++++++++++------------- tests/unit/test_environment.py | 32 ++++++++++++++++ tests/unit/test_validator.py | 2 +- 8 files changed, 218 insertions(+), 38 deletions(-) create mode 100644 datumaro/components/lazy_plugin.py create mode 100644 datumaro/plugins/specs.json create mode 100644 datumaro/plugins/specs.py create mode 100644 tests/unit/test_environment.py diff --git a/datumaro/components/dataset.py b/datumaro/components/dataset.py index 615cb0d861..91d2b3d2e0 100644 --- a/datumaro/components/dataset.py +++ b/datumaro/components/dataset.py @@ -48,7 +48,6 @@ from datumaro.components.merge import DEFAULT_MERGE_POLICY from datumaro.components.progress_reporting import NullProgressReporter, ProgressReporter from datumaro.components.transformer import ItemTransform, Transform -from datumaro.plugins.transforms import ProjectLabels from datumaro.util import is_method_redefined from datumaro.util.log_utils import logging_disabled from datumaro.util.meta_file_util import load_hash_key @@ -626,6 +625,8 @@ def update(self, source: Union[DatasetPatch, IDataset, Iterable[DatasetItem]]): else: self.put(source.data.get(*item_id)) elif isinstance(source, IDataset): + from datumaro.plugins.transforms import ProjectLabels + for item in ProjectLabels( source, self.categories().get(AnnotationType.label, LabelCategories()) ): diff --git a/datumaro/components/environment.py b/datumaro/components/environment.py index a064663c8f..46e9bb710e 100644 --- a/datumaro/components/environment.py +++ b/datumaro/components/environment.py @@ -6,11 +6,13 @@ import importlib import logging as log import os.path as osp +from contextlib import contextmanager from functools import partial from inspect import isclass from typing import ( Callable, Dict, + Generator, Generic, Iterable, Iterator, @@ -18,6 +20,7 @@ Optional, Sequence, Set, + Tuple, Type, TypeVar, ) @@ -29,6 +32,7 @@ RejectionReason, detect_dataset_format, ) +from datumaro.components.lazy_plugin import PLUGIN_TYPES, LazyPlugin from datumaro.util.os_util import import_foreign_module, split_path T = TypeVar("T") @@ -36,27 +40,31 @@ class Registry(Generic[T]): def __init__(self): - self.items: Dict[str, T] = {} + self._items: Dict[str, T] = {} def register(self, name: str, value: T) -> T: - self.items[name] = value + self._items[name] = value return value def unregister(self, name: str) -> Optional[T]: - return self.items.pop(name, None) + return self._items.pop(name, None) def get(self, key: str): """Returns a class or a factory function""" - return self.items[key] + return self._items[key] def __getitem__(self, key: str) -> T: return self.get(key) def __contains__(self, key) -> bool: - return key in self.items + return key in self._items def __iter__(self) -> Iterator[str]: - return iter(self.items) + return iter(self._items) + + def items(self) -> Generator[Tuple[str, T], None, None]: + for key in self: + yield key, self.get(key) class PluginRegistry(Registry[Type[CliPlugin]]): @@ -66,6 +74,13 @@ def __init__( super().__init__() self._filter = filter + def get(self, key: str) -> PLUGIN_TYPES: + """Returns a class or a factory function""" + item = self._items[key] + if issubclass(item, LazyPlugin): + return item.get_plugin_cls() + return item + def batch_register(self, values: Iterable[CliPlugin]): for v in values: if self._filter and not self._filter(v): @@ -88,7 +103,7 @@ def _make_filter(cls, accept, skip=None): def _check_type(t, *, accept, skip): return issubclass(t, accept) and t not in skip - def __init__(self): + def __init__(self, use_lazy_import: bool = True): from datumaro.components.dataset_base import DatasetBase, SubsetBase from datumaro.components.exporter import Exporter from datumaro.components.generator import DatasetGenerator @@ -98,7 +113,9 @@ def __init__(self): from datumaro.components.validator import Validator _filter = self._make_filter - self._extractors = PluginRegistry(_filter(DatasetBase, skip=SubsetBase)) + self._extractors = PluginRegistry( + _filter(DatasetBase, skip=(SubsetBase, Transform, ItemTransform)) + ) self._importers = PluginRegistry(_filter(Importer)) self._launchers = PluginRegistry(_filter(Launcher)) self._exporters = PluginRegistry(_filter(Exporter)) @@ -106,6 +123,7 @@ def __init__(self): self._transforms = PluginRegistry(_filter(Transform, skip=ItemTransform)) self._validators = PluginRegistry(_filter(Validator)) self._builtins_initialized = False + self._use_lazy_import = use_lazy_import def _get_plugin_registry(self, name): if not self._builtins_initialized: @@ -210,6 +228,16 @@ def _load_plugins(cls, module_names, *, importer, types=None): @classmethod def _load_builtin_plugins(cls): + """Load builtin plugins which will be imported lazily using plugin spec files""" + if cls._builtin_plugins is None: + from datumaro.plugins.specs import LAZY_PLUGINS + + cls._builtin_plugins = LAZY_PLUGINS + return cls._builtin_plugins + + @classmethod + def _load_builtin_plugins_from_importlib(cls): + """Load builtin plugins from importlib, not lazy import from plugin spec files""" if cls._builtin_plugins is None: import datumaro.plugins @@ -229,7 +257,11 @@ def load_plugins(self, plugins_dir): self.register_plugins(plugins) def _register_builtin_plugins(self): - self.register_plugins(self._load_builtin_plugins()) + self.register_plugins( + self._load_builtin_plugins() + if self._use_lazy_import + else self._load_builtin_plugins_from_importlib() + ) def register_plugins(self, plugins): self.extractors.batch_register(plugins) @@ -274,7 +306,7 @@ def detect_dataset( detected_formats = detect_dataset_format( ( (format_name, importer.detect) - for format_name, importer in self.importers.items.items() + for format_name, importer in self.importers.items() ), path, rejection_callback=rejection_callback, @@ -322,5 +354,11 @@ def _register(registry: PluginRegistry): return merged + @classmethod + @contextmanager + def release_builtin_plugins(cls): + cls._builtin_plugins = None + yield + DEFAULT_ENVIRONMENT = Environment() diff --git a/datumaro/components/lazy_plugin.py b/datumaro/components/lazy_plugin.py new file mode 100644 index 0000000000..ff797443b7 --- /dev/null +++ b/datumaro/components/lazy_plugin.py @@ -0,0 +1,59 @@ +# Copyright (C) 2023 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from abc import ABC, abstractclassmethod +from importlib import import_module +from typing import Type, Union, get_args + +from datumaro.components.dataset_base import DatasetBase +from datumaro.components.exporter import Exporter +from datumaro.components.generator import DatasetGenerator +from datumaro.components.importer import Importer +from datumaro.components.launcher import Launcher +from datumaro.components.transformer import Transform +from datumaro.components.validator import Validator + +# Transform should be in front of DatasetBase, +# since Transform inherits DatasetBase. +_PLUGIN_TYPES = Union[ + Transform, + Exporter, + DatasetGenerator, + Importer, + Launcher, + Validator, + DatasetBase, +] +PLUGIN_TYPES = Type[_PLUGIN_TYPES] +STR_TO_PLUGIN_TYPES = {t.__name__: t for t in get_args(_PLUGIN_TYPES)} + + +class LazyPlugin(ABC): + NAME: str + + @abstractclassmethod + def get_plugin_cls(cls) -> PLUGIN_TYPES: + pass + + +def get_lazy_plugin( + import_path: str, + plugin_name: str, + plugin_type: str, +) -> LazyPlugin: + plugin_type_cls = STR_TO_PLUGIN_TYPES[plugin_type] + + class LazyPluginImpl(LazyPlugin, plugin_type_cls): + NAME = plugin_name + + @classmethod + def get_plugin_cls(cls) -> PLUGIN_TYPES: + splits = import_path.split(".") + module_name = ".".join(splits[:-1]) + class_name = splits[-1] + module = import_module(module_name) + plugin_cls = getattr(module, class_name) + return plugin_cls + + return LazyPluginImpl diff --git a/datumaro/plugins/specs.json b/datumaro/plugins/specs.json new file mode 100644 index 0000000000..24430036f4 --- /dev/null +++ b/datumaro/plugins/specs.json @@ -0,0 +1 @@ +[{"import_path":"datumaro.plugins.transforms.AnnsToLabels","plugin_name":"anns_to_labels","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.BboxValuesDecrement","plugin_name":"bbox_values_decrement","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.BoxesToMasks","plugin_name":"boxes_to_masks","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.Correct","plugin_name":"correct","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.CropCoveredSegments","plugin_name":"crop_covered_segments","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.IdFromImageName","plugin_name":"id_from_image_name","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.MapSubsets","plugin_name":"map_subsets","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.MasksToPolygons","plugin_name":"masks_to_polygons","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.MergeInstanceSegments","plugin_name":"merge_instance_segments","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.PolygonsToMasks","plugin_name":"polygons_to_masks","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ProjectInfos","plugin_name":"project_infos","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ProjectLabels","plugin_name":"project_labels","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RandomSplit","plugin_name":"random_split","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.Reindex","plugin_name":"reindex","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ReindexAnnotations","plugin_name":"reindex_annotations","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RemapLabels","plugin_name":"remap_labels","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RemoveAnnotations","plugin_name":"remove_annotations","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RemoveAttributes","plugin_name":"remove_attributes","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RemoveItems","plugin_name":"remove_items","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.Rename","plugin_name":"rename","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ResizeTransform","plugin_name":"resize","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ShapesToBoxes","plugin_name":"shapes_to_boxes","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.Sort","plugin_name":"sort","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.splitter.Split","plugin_name":"split","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.ndr.NDR","plugin_name":"ndr","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesBase","plugin_name":"cityscapes","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.ade20k2020.Ade20k2020Base","plugin_name":"ade20k2020","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidBase","plugin_name":"camvid","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirBase","plugin_name":"image_dir","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.lfw.LfwBase","plugin_name":"lfw","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.vott_json.VottJsonBase","plugin_name":"vott_json","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.common_super_resolution.CommonSuperResolutionBase","plugin_name":"common_super_resolution","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.kinetics.KineticsBase","plugin_name":"kinetics","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtBase","plugin_name":"imagenet_txt","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mots.MotsPngExtractor","plugin_name":"mots_png_extractor","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.ade20k2017.Ade20k2017Base","plugin_name":"ade20k2017","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.nyu_depth_v2.NyuDepthV2Base","plugin_name":"nyu_depth_v2","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mnist.MnistBase","plugin_name":"mnist","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetBase","plugin_name":"imagenet","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceBase","plugin_name":"wider_face","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.brats.BratsBase","plugin_name":"brats","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Base","plugin_name":"vgg_face2","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.video.VideoFramesBase","plugin_name":"video_frames","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.video.VideoKeyframesBase","plugin_name":"video_keyframes","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mars.MarsBase","plugin_name":"mars","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.brats_numpy.BratsNumpyBase","plugin_name":"brats_numpy","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipBase","plugin_name":"image_zip","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.cifar.CifarBase","plugin_name":"cifar","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationBase","plugin_name":"common_semantic_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqBase","plugin_name":"mot_seq","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeBase","plugin_name":"label_me","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvBase","plugin_name":"mnist_csv","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Base","plugin_name":"market1501","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesBase","plugin_name":"open_images","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.vott_csv.VottCsvBase","plugin_name":"vott_csv","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.tiling.tile.Tile","plugin_name":"tile","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.tiling.merge_tile.MergeTile","plugin_name":"merge_tile","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.sampler.random_sampler.LabelRandomSampler","plugin_name":"label_random_sampler","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.sampler.random_sampler.RandomSampler","plugin_name":"random_sampler","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.sampler.relevancy_sampler.RelevancySampler","plugin_name":"relevancy_sampler","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.celeba.celeba.CelebaBase","plugin_name":"celeba","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.celeba.align_celeba.AlignCelebaBase","plugin_name":"align_celeba","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.segment_anything.base.SegmentAnythingBase","plugin_name":"segment_anything","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.base.SuperviselyPointCloudBase","plugin_name":"sly_pointcloud","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextLocalizationBase","plugin_name":"icdar_text_localization","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextSegmentationBase","plugin_name":"icdar_text_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarWordRecognitionBase","plugin_name":"icdar_word_recognition","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloLooseBase","plugin_name":"yolo_loose","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloStrictBase","plugin_name":"yolo_strict","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloUltralyticsBase","plugin_name":"yolo_ultralytics","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_mat.MpiiBase","plugin_name":"mpii","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_json.MpiiJsonBase","plugin_name":"mpii_json","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaBase","plugin_name":"ava","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocActionBase","plugin_name":"voc_action","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocBase","plugin_name":"voc","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocClassificationBase","plugin_name":"voc_classification","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocDetectionBase","plugin_name":"voc_detection","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocInstanceSegmentationBase","plugin_name":"voc_instance_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocLayoutBase","plugin_name":"voc_layout","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocSegmentationBase","plugin_name":"voc_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoCaptionsBase","plugin_name":"coco_captions","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoImageInfoBase","plugin_name":"coco_image_info","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoInstancesBase","plugin_name":"coco_instances","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoLabelsBase","plugin_name":"coco_labels","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoPanopticBase","plugin_name":"coco_panoptic","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoPersonKeypointsBase","plugin_name":"coco_person_keypoints","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoStuffBase","plugin_name":"coco_stuff","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.arrow.base.ArrowBase","plugin_name":"arrow","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.datumaro.base.DatumaroBase","plugin_name":"datumaro","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaAlBase","plugin_name":"synthia_al","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaRandBase","plugin_name":"synthia_rand","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaSfBase","plugin_name":"synthia_sf","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecClassificationBase","plugin_name":"mvtec_classification","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecDetectionBase","plugin_name":"mvtec_detection","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecSegmentationBase","plugin_name":"mvtec_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.cvat.base.CvatBase","plugin_name":"cvat","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.kitti_raw.base.KittiRawBase","plugin_name":"kitti_raw","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.base.TfDetectionApiBase","plugin_name":"tf_detection_api","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.kitti.base.KittiDetectionBase","plugin_name":"kitti_detection","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.kitti.base.KittiSegmentationBase","plugin_name":"kitti_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.base.DatumaroBinaryBase","plugin_name":"datumaro_binary","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.base.MapillaryVistasInstancesBase","plugin_name":"mapillary_vistas_instances","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.base.MapillaryVistasPanopticBase","plugin_name":"mapillary_vistas_panoptic","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesImporter","plugin_name":"cityscapes","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.ade20k2020.Ade20k2020Importer","plugin_name":"ade20k2020","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidImporter","plugin_name":"camvid","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirImporter","plugin_name":"image_dir","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.lfw.LfwImporter","plugin_name":"lfw","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.vott_json.VottJsonImporter","plugin_name":"vott_json","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.common_super_resolution.CommonSuperResolutionImporter","plugin_name":"common_super_resolution","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kinetics.KineticsImporter","plugin_name":"kinetics","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtImporter","plugin_name":"imagenet_txt","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mots.MotsImporter","plugin_name":"mots","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.ade20k2017.Ade20k2017Importer","plugin_name":"ade20k2017","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.nyu_depth_v2.NyuDepthV2Importer","plugin_name":"nyu_depth_v2","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mnist.MnistImporter","plugin_name":"mnist","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetImporter","plugin_name":"imagenet","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetWithSubsetDirsImporter","plugin_name":"imagenet_with_subset_dirs","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceImporter","plugin_name":"wider_face","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.brats.BratsImporter","plugin_name":"brats","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Importer","plugin_name":"vgg_face2","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.video.VideoFramesImporter","plugin_name":"video_frames","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.video.VideoKeyframesImporter","plugin_name":"video_keyframes","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mars.MarsImporter","plugin_name":"mars","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.brats_numpy.BratsNumpyImporter","plugin_name":"brats_numpy","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipImporter","plugin_name":"image_zip","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.cifar.CifarImporter","plugin_name":"cifar","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationImporter","plugin_name":"common_semantic_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationWithSubsetDirsImporter","plugin_name":"common_semantic_segmentation_with_subset_dirs","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqImporter","plugin_name":"mot_seq","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeImporter","plugin_name":"label_me","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvImporter","plugin_name":"mnist_csv","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Importer","plugin_name":"market1501","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesImporter","plugin_name":"open_images","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.vott_csv.VottCsvImporter","plugin_name":"vott_csv","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.celeba.celeba.CelebaImporter","plugin_name":"celeba","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.celeba.align_celeba.AlignCelebaImporter","plugin_name":"align_celeba","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.segment_anything.importer.SegmentAnythingImporter","plugin_name":"segment_anything","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.base.SuperviselyPointCloudImporter","plugin_name":"sly_pointcloud","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextLocalizationImporter","plugin_name":"icdar_text_localization","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextSegmentationImporter","plugin_name":"icdar_text_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarWordRecognitionImporter","plugin_name":"icdar_word_recognition","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.yolo.importer.YoloImporter","plugin_name":"yolo","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_mat.MpiiImporter","plugin_name":"mpii","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_json.MpiiJsonImporter","plugin_name":"mpii_json","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaImporter","plugin_name":"ava","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocActionImporter","plugin_name":"voc_action","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocClassificationImporter","plugin_name":"voc_classification","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocDetectionImporter","plugin_name":"voc_detection","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocImporter","plugin_name":"voc","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocInstanceSegmentationImporter","plugin_name":"voc_instance_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocLayoutImporter","plugin_name":"voc_layout","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocSegmentationImporter","plugin_name":"voc_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoCaptionsImporter","plugin_name":"coco_captions","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoImageInfoImporter","plugin_name":"coco_image_info","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoImporter","plugin_name":"coco","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoInstancesImporter","plugin_name":"coco_instances","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoLabelsImporter","plugin_name":"coco_labels","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoPanopticImporter","plugin_name":"coco_panoptic","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoPersonKeypointsImporter","plugin_name":"coco_person_keypoints","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoRoboflowImporter","plugin_name":"coco_roboflow","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoStuffImporter","plugin_name":"coco_stuff","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.arrow.importer.ArrowImporter","plugin_name":"arrow","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaAlImporter","plugin_name":"synthia_al","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaRandImporter","plugin_name":"synthia_rand","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaSfImporter","plugin_name":"synthia_sf","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecClassificationImporter","plugin_name":"mvtec_classification","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecDetectionImporter","plugin_name":"mvtec_detection","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecImporter","plugin_name":"mvtec","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecSegmentationImporter","plugin_name":"mvtec_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.cvat.base.CvatImporter","plugin_name":"cvat","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kitti_raw.base.KittiRawImporter","plugin_name":"kitti_raw","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.base.TfDetectionApiImporter","plugin_name":"tf_detection_api","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.datumaro.importer.DatumaroImporter","plugin_name":"datumaro","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiDetectionImporter","plugin_name":"kitti_detection","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiImporter","plugin_name":"kitti","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiSegmentationImporter","plugin_name":"kitti_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.importer.DatumaroBinaryImporter","plugin_name":"datumaro_binary","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasImporter","plugin_name":"mapillary_vistas","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasInstancesImporter","plugin_name":"mapillary_vistas_instances","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasPanopticImporter","plugin_name":"mapillary_vistas_panoptic","plugin_type":"Importer"},{"import_path":"datumaro.plugins.explorer.ExplorerLauncher","plugin_name":"explorer","plugin_type":"Launcher"},{"import_path":"datumaro.plugins.openvino_plugin.launcher.OpenvinoLauncher","plugin_name":"openvino","plugin_type":"Launcher"},{"import_path":"datumaro.plugins.shift_analyzer.ShiftAnalyzerLauncher","plugin_name":"shift_analyzer","plugin_type":"Launcher"},{"import_path":"datumaro.plugins.accuracy_checker_plugin.ac_launcher.AcLauncher","plugin_name":"ac","plugin_type":"Launcher"},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesExporter","plugin_name":"cityscapes","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidExporter","plugin_name":"camvid","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirExporter","plugin_name":"image_dir","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.lfw.LfwExporter","plugin_name":"lfw","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtExporter","plugin_name":"imagenet_txt","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mots.MotsPngExporter","plugin_name":"mots_png","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mnist.MnistExporter","plugin_name":"mnist","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetExporter","plugin_name":"imagenet","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetWithSubsetDirsExporter","plugin_name":"imagenet_with_subset_dirs","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceExporter","plugin_name":"wider_face","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Exporter","plugin_name":"vgg_face2","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipExporter","plugin_name":"image_zip","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.cifar.CifarExporter","plugin_name":"cifar","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqGtExporter","plugin_name":"mot_seq_gt","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeExporter","plugin_name":"label_me","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvExporter","plugin_name":"mnist_csv","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Exporter","plugin_name":"market1501","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesExporter","plugin_name":"open_images","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.segment_anything.exporter.SegmentAnythingExporter","plugin_name":"segment_anything","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.exporter.SuperviselyPointCloudExporter","plugin_name":"sly_pointcloud","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarTextLocalizationExporter","plugin_name":"icdar_text_localization","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarTextSegmentationExporter","plugin_name":"icdar_text_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarWordRecognitionExporter","plugin_name":"icdar_word_recognition","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.yolo.exporter.YoloExporter","plugin_name":"yolo","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.yolo.exporter.YoloUltralyticsExporter","plugin_name":"yolo_ultralytics","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaExporter","plugin_name":"ava","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocActionExporter","plugin_name":"voc_action","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocClassificationExporter","plugin_name":"voc_classification","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocDetectionExporter","plugin_name":"voc_detection","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocExporter","plugin_name":"voc","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocInstanceSegmentationExporter","plugin_name":"voc_instance_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocLayoutExporter","plugin_name":"voc_layout","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocSegmentationExporter","plugin_name":"voc_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoCaptionsExporter","plugin_name":"coco_captions","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoExporter","plugin_name":"coco","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoImageInfoExporter","plugin_name":"coco_image_info","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoInstancesExporter","plugin_name":"coco_instances","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoLabelsExporter","plugin_name":"coco_labels","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoPanopticExporter","plugin_name":"coco_panoptic","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoPersonKeypointsExporter","plugin_name":"coco_person_keypoints","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoStuffExporter","plugin_name":"coco_stuff","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.arrow.exporter.ArrowExporter","plugin_name":"arrow","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecClassificationExporter","plugin_name":"mvtec_classification","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecDetectionExporter","plugin_name":"mvtec_detection","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecExporter","plugin_name":"mvtec","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecSegmentationExporter","plugin_name":"mvtec_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.cvat.exporter.CvatExporter","plugin_name":"cvat","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.kitti_raw.exporter.KittiRawExporter","plugin_name":"kitti_raw","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.exporter.TfDetectionApiExporter","plugin_name":"tf_detection_api","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.datumaro.exporter.DatumaroExporter","plugin_name":"datumaro","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiDetectionExporter","plugin_name":"kitti_detection","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiExporter","plugin_name":"kitti","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiSegmentationExporter","plugin_name":"kitti_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.exporter.DatumaroBinaryExporter","plugin_name":"datumaro_binary","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.transforms.AnnsToLabels","plugin_name":"anns_to_labels","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.BboxValuesDecrement","plugin_name":"bbox_values_decrement","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.BoxesToMasks","plugin_name":"boxes_to_masks","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.Correct","plugin_name":"correct","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.CropCoveredSegments","plugin_name":"crop_covered_segments","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.IdFromImageName","plugin_name":"id_from_image_name","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.MapSubsets","plugin_name":"map_subsets","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.MasksToPolygons","plugin_name":"masks_to_polygons","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.MergeInstanceSegments","plugin_name":"merge_instance_segments","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.PolygonsToMasks","plugin_name":"polygons_to_masks","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ProjectInfos","plugin_name":"project_infos","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ProjectLabels","plugin_name":"project_labels","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RandomSplit","plugin_name":"random_split","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.Reindex","plugin_name":"reindex","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ReindexAnnotations","plugin_name":"reindex_annotations","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RemapLabels","plugin_name":"remap_labels","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RemoveAnnotations","plugin_name":"remove_annotations","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RemoveAttributes","plugin_name":"remove_attributes","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RemoveItems","plugin_name":"remove_items","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.Rename","plugin_name":"rename","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ResizeTransform","plugin_name":"resize","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ShapesToBoxes","plugin_name":"shapes_to_boxes","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.Sort","plugin_name":"sort","plugin_type":"Transform"},{"import_path":"datumaro.plugins.splitter.Split","plugin_name":"split","plugin_type":"Transform"},{"import_path":"datumaro.plugins.ndr.NDR","plugin_name":"ndr","plugin_type":"Transform"},{"import_path":"datumaro.plugins.tiling.tile.Tile","plugin_name":"tile","plugin_type":"Transform"},{"import_path":"datumaro.plugins.tiling.merge_tile.MergeTile","plugin_name":"merge_tile","plugin_type":"Transform"},{"import_path":"datumaro.plugins.sampler.random_sampler.LabelRandomSampler","plugin_name":"label_random_sampler","plugin_type":"Transform"},{"import_path":"datumaro.plugins.sampler.random_sampler.RandomSampler","plugin_name":"random_sampler","plugin_type":"Transform"},{"import_path":"datumaro.plugins.sampler.relevancy_sampler.RelevancySampler","plugin_name":"relevancy_sampler","plugin_type":"Transform"},{"import_path":"datumaro.plugins.validators.ClassificationValidator","plugin_name":"classification","plugin_type":"Validator"},{"import_path":"datumaro.plugins.validators.DetectionValidator","plugin_name":"detection","plugin_type":"Validator"},{"import_path":"datumaro.plugins.validators.SegmentationValidator","plugin_name":"segmentation","plugin_type":"Validator"}] diff --git a/datumaro/plugins/specs.py b/datumaro/plugins/specs.py new file mode 100644 index 0000000000..cb707e8e23 --- /dev/null +++ b/datumaro/plugins/specs.py @@ -0,0 +1,49 @@ +# Copyright (C) 2023 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from pathlib import Path + +from datumaro.components.lazy_plugin import get_lazy_plugin +from datumaro.util import parse_json_file + +_SOURCE_PATH = Path(__file__).resolve() +_SOURCE_DIR = _SOURCE_PATH.parent +_SPECS_JSON_PATH = _SOURCE_DIR / "specs.json" + +LAZY_PLUGINS = [ + get_lazy_plugin(spec["import_path"], spec["plugin_name"], spec["plugin_type"]) + for spec in parse_json_file(str(_SPECS_JSON_PATH)) +] + +if __name__ == "__main__": + from datumaro.components.environment import Environment + from datumaro.util import dump_json_file + + env = Environment(use_lazy_import=False) + plugin_specs = [] + + def _enroll_to_plugin_specs(plugins, plugin_type): + global plugin_specs + + for _, plugin in plugins.items(): + mod = plugin.__module__ + class_name = plugin.__name__ + plugin_name = plugin.NAME + plugin_specs += [ + { + "import_path": f"{mod}.{class_name}", + "plugin_name": plugin_name, + "plugin_type": plugin_type, + } + ] + + _enroll_to_plugin_specs(env.extractors, "DatasetBase") + _enroll_to_plugin_specs(env.importers, "Importer") + _enroll_to_plugin_specs(env.launchers, "Launcher") + _enroll_to_plugin_specs(env.exporters, "Exporter") + _enroll_to_plugin_specs(env.generators, "DatasetGenerator") + _enroll_to_plugin_specs(env.transforms, "Transform") + _enroll_to_plugin_specs(env.validators, "Validator") + + dump_json_file(_SPECS_JSON_PATH, plugin_specs) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index a72dd180ed..d07837c83a 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -227,8 +227,8 @@ def test_can_save_and_load(self): @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self): env = Environment() - env.importers.items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} - env.extractors.items = {DEFAULT_FORMAT: env.extractors[DEFAULT_FORMAT]} + env.importers._items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} + env.extractors._items = {DEFAULT_FORMAT: env.extractors[DEFAULT_FORMAT]} dataset = Dataset.from_iterable( [ @@ -247,8 +247,8 @@ def test_can_detect(self): @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_with_nested_folder(self): env = Environment() - env.importers.items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} - env.extractors.items = {DEFAULT_FORMAT: env.extractors[DEFAULT_FORMAT]} + env.importers._items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} + env.extractors._items = {DEFAULT_FORMAT: env.extractors[DEFAULT_FORMAT]} dataset = Dataset.from_iterable( [ @@ -295,8 +295,8 @@ def test_cannot_detect_for_non_existent_path(self): @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect_and_import(self): env = Environment() - env.importers.items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} - env.extractors.items = {DEFAULT_FORMAT: env.extractors[DEFAULT_FORMAT]} + env.importers._items = {DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT]} + env.extractors._items = {DEFAULT_FORMAT: env.extractors[DEFAULT_FORMAT]} source_dataset = Dataset.from_iterable( [ @@ -317,10 +317,10 @@ def test_can_detect_and_import(self): @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_report_no_dataset_found(self): env = Environment() - env.importers.items = { + env.importers._items = { DEFAULT_FORMAT: env.importers[DEFAULT_FORMAT], } - env.extractors.items = { + env.extractors._items = { DEFAULT_FORMAT: env.extractors[DEFAULT_FORMAT], } @@ -330,11 +330,11 @@ def test_can_report_no_dataset_found(self): @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_report_multiple_formats_match(self): env = Environment() - env.importers.items = { + env.importers._items = { "a": env.importers[DEFAULT_FORMAT], "b": env.importers[DEFAULT_FORMAT], } - env.extractors.items = { + env.extractors._items = { "a": env.extractors[DEFAULT_FORMAT], "b": env.extractors[DEFAULT_FORMAT], } @@ -355,8 +355,8 @@ def test_can_report_multiple_formats_match(self): @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_report_no_matching_formats(self): env = Environment() - env.importers.items = {} - env.extractors.items = {} + env.importers._items = {} + env.extractors._items = {} source_dataset = Dataset.from_iterable( [ @@ -374,8 +374,8 @@ def test_can_report_no_matching_formats(self): @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_report_unknown_format_requested(self): env = Environment() - env.importers.items = {} - env.extractors.items = {} + env.importers._items = {} + env.extractors._items = {} source_dataset = Dataset.from_iterable( [ @@ -393,7 +393,7 @@ def test_can_report_unknown_format_requested(self): @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_export_by_string_format_name(self): env = Environment() - env.exporters.items = {"qq": env.exporters[DEFAULT_FORMAT]} + env.exporters._items = {"qq": env.exporters[DEFAULT_FORMAT]} dataset = Dataset.from_iterable( [ @@ -1623,7 +1623,7 @@ def _apply_impl(self): self._save_image(item, name=name) env = Environment() - env.exporters.items = {"test": CustomExporter} + env.exporters._items = {"test": CustomExporter} with TestDir() as path: dataset = Dataset.from_iterable( @@ -1759,8 +1759,8 @@ def __iter__(self): yield DatasetItem("1") env = Environment() - env.importers.items.clear() - env.extractors.items["test"] = TestExtractor + env.importers._items.clear() + env.extractors._items["test"] = TestExtractor dataset = Dataset.import_from("", "test", env=env, progress_reporter=NullProgressReporter()) @@ -1776,8 +1776,8 @@ def __iter__(self): yield DatasetItem("1") env = Environment() - env.importers.items.clear() - env.extractors.items["test"] = TestExtractor + env.importers._items.clear() + env.extractors._items["test"] = TestExtractor dataset = Dataset.import_from("", "test", env=env, error_policy=FailingImportErrorPolicy()) @@ -1805,8 +1805,8 @@ def split(self, count): progress_reporter.finish = mock.MagicMock() env = Environment() - env.importers.items.clear() - env.extractors.items["test"] = TestExtractor + env.importers._items.clear() + env.extractors._items["test"] = TestExtractor Dataset.import_from("", "test", env=env, progress_reporter=progress_reporter) @@ -1843,8 +1843,8 @@ def split(self, count): ) env = Environment() - env.importers.items.clear() - env.extractors.items["test"] = TestExtractor + env.importers._items.clear() + env.extractors._items["test"] = TestExtractor Dataset.import_from("", "test", env=env, progress_reporter=progress_reporter) @@ -1865,8 +1865,8 @@ class TestError(Exception): yield from [] env = Environment() - env.importers.items.clear() - env.extractors.items["test"] = TestExtractor + env.importers._items.clear() + env.extractors._items["test"] = TestExtractor class TestErrorPolicy(ImportErrorPolicy): pass diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py new file mode 100644 index 0000000000..27f1ce353f --- /dev/null +++ b/tests/unit/test_environment.py @@ -0,0 +1,32 @@ +# Copyright (C) 2019-2023 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import pytest + +from datumaro.components.environment import Environment + + +class EnvironmentTest: + @pytest.fixture + def fxt_lazy_import(self): + with Environment.release_builtin_plugins(): + env = Environment(use_lazy_import=True) + _ = env.importers + yield env + + @pytest.fixture + def fxt_no_lazy_import(self): + with Environment.release_builtin_plugins(): + env = Environment(use_lazy_import=False) + _ = env.importers + yield env + + def test_equivalance(self, fxt_lazy_import: Environment, fxt_no_lazy_import: Environment): + assert sorted(fxt_lazy_import.extractors) == sorted(fxt_no_lazy_import.extractors) + assert sorted(fxt_lazy_import.importers) == sorted(fxt_no_lazy_import.importers) + assert sorted(fxt_lazy_import.launchers) == sorted(fxt_no_lazy_import.launchers) + assert sorted(fxt_lazy_import.exporters) == sorted(fxt_no_lazy_import.exporters) + assert sorted(fxt_lazy_import.generators) == sorted(fxt_no_lazy_import.generators) + assert sorted(fxt_lazy_import.transforms) == sorted(fxt_no_lazy_import.transforms) + assert sorted(fxt_lazy_import.validators) == sorted(fxt_no_lazy_import.validators) diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py index a387ae2116..ca43a5e4f4 100644 --- a/tests/unit/test_validator.py +++ b/tests/unit/test_validator.py @@ -376,7 +376,7 @@ def setUpClass(cls): def test_has_enum_entries_in_environment(self): env = Environment() for key in TaskType.__members__: - self.assertIn(key, env.validators.items) + self.assertIn(key, env.validators._items) @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_generate_reports(self): From 48aeb648cc031d105052e20681e359f9bd28798a Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Tue, 30 May 2023 18:11:37 +0900 Subject: [PATCH 03/10] Add datumaro/plugins/README.md Signed-off-by: Kim, Vinnam --- datumaro/plugins/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 datumaro/plugins/README.md diff --git a/datumaro/plugins/README.md b/datumaro/plugins/README.md new file mode 100644 index 0000000000..dc4d182ea1 --- /dev/null +++ b/datumaro/plugins/README.md @@ -0,0 +1,11 @@ +# Add a new builtin plugin + +If you are adding a new builtin plugin under `./datumaro/plugins/`, you have to enroll it to `./datumaro/plugins/specs.json` for the lazy import. +Otherwise, you will be failed in this test (`tests/unit/test_environment.py`). +You can enroll your plugin to `./datumaro/plugins/specs.json` with Python executable script as follows. + +```console +python datumaro/plugins/specs.py +``` + +> **_NOTE:_** It is not recommended to mannually modify `./datumaro/plugins/specs.json` by hands. Please use the above Python executable script. From 4514394dcc362c065ed3f1df3d9553f6d9fbe766 Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Tue, 30 May 2023 18:22:09 +0900 Subject: [PATCH 04/10] Update CHANGELOG.md Signed-off-by: Kim, Vinnam --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c1c9d5b4..bac4f7ba25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## \[Unreleased\] +### New features +### Enhancements +- Enhance import performance for built-in plugins + () +### Bug fixes ## 26/05/2023 - Release 1.3.1 ### Bug fixes From e41ed737d4680f798f7beffac9a4dc6642418f26 Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Tue, 30 May 2023 18:43:34 +0900 Subject: [PATCH 05/10] Update MANIFEST.in Signed-off-by: Kim, Vinnam --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index b8c194ac1c..2cf27f7bb5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include README.md include requirements-core.txt include requirements-default.txt include 3rd-party.txt +include datumaro/plugins/specs.json From 6e6d1d616d77696eee37c2b1222d5e05256eb7f8 Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Tue, 30 May 2023 21:25:55 +0900 Subject: [PATCH 06/10] Add openvino-dev to default extra deps and update contributing.md Signed-off-by: Kim, Vinnam --- contributing.md | 15 ++++++++------- requirements-default.txt | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/contributing.md b/contributing.md index 1af2c2269d..4c660c7637 100644 --- a/contributing.md +++ b/contributing.md @@ -5,7 +5,7 @@ - [Design document](https://openvinotoolkit.github.io/datumaro/latest/docs/explanation/architecture) - [Developer manual](https://openvinotoolkit.github.io/datumaro/latest/docs/reference/datumaro_module) -## Installation +## Installation for developer ### Prerequisites @@ -23,20 +23,21 @@ python -m virtualenv venv . venv/bin/activate ``` -Then install all dependencies: - +Install Datumaro with optional dependencies: ``` bash -pip install -r requirements.txt +cd /path/to/the/cloned/repo/ +pip install -e .[default, tf] ``` -Install Datumaro: +Then install test dependencies: + ``` bash -pip install -e /path/to/the/cloned/repo/ +pip install -r tests/requirements.txt ``` **Optional dependencies** -These components are only required for plugins and not installed by default: +Developer should install the following optional components for running our tests: - OpenVINO - Accuracy Checker diff --git a/requirements-default.txt b/requirements-default.txt index ba094413bd..e52519199e 100644 --- a/requirements-default.txt +++ b/requirements-default.txt @@ -2,3 +2,4 @@ dvc>=2.7.0 fsspec <= 2022.11.0; python_version < '3.8' # https://github.com/openvinotoolkit/datumaro/actions/runs/4003215621/jobs/6871114851#step:5:1647 GitPython>=3.1.18,!=3.1.25 # https://github.com/openvinotoolkit/datumaro/issues/612 openvino-telemetry>=2022.1.0 +openvino-dev==2022.3.0 From d8f1fd0055fead402d7dde7a5000fa9271b165cb Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Wed, 31 May 2023 16:14:41 +0900 Subject: [PATCH 07/10] Add extra_deps mechanism to lazy plugin import Signed-off-by: Kim, Vinnam --- datumaro/components/lazy_plugin.py | 40 ++++++++++++++++++- datumaro/plugins/README.md | 20 +++++++++- .../accuracy_checker_plugin/ac_launcher.py | 2 + .../data_formats/tf_detection_api/base.py | 3 ++ .../data_formats/tf_detection_api/exporter.py | 2 + datumaro/plugins/specs.json | 2 +- datumaro/plugins/specs.py | 13 ++++-- tests/unit/test_environment.py | 27 +++++++++++++ 8 files changed, 102 insertions(+), 7 deletions(-) diff --git a/datumaro/components/lazy_plugin.py b/datumaro/components/lazy_plugin.py index ff797443b7..1a01939b27 100644 --- a/datumaro/components/lazy_plugin.py +++ b/datumaro/components/lazy_plugin.py @@ -4,9 +4,10 @@ from abc import ABC, abstractclassmethod from importlib import import_module -from typing import Type, Union, get_args +from typing import List, Optional, Sequence, Type, Union, get_args from datumaro.components.dataset_base import DatasetBase +from datumaro.components.errors import DatumaroError from datumaro.components.exporter import Exporter from datumaro.components.generator import DatasetGenerator from datumaro.components.importer import Importer @@ -27,6 +28,7 @@ ] PLUGIN_TYPES = Type[_PLUGIN_TYPES] STR_TO_PLUGIN_TYPES = {t.__name__: t for t in get_args(_PLUGIN_TYPES)} +_EXTRA_DEPS_ATTR_NAME = "__extra_deps__" class LazyPlugin(ABC): @@ -41,7 +43,14 @@ def get_lazy_plugin( import_path: str, plugin_name: str, plugin_type: str, -) -> LazyPlugin: + extra_deps: List[str] = [], +) -> Optional[LazyPlugin]: + try: + for extra_dep in extra_deps: + import_module(extra_dep) + except ImportError: + return None + plugin_type_cls = STR_TO_PLUGIN_TYPES[plugin_type] class LazyPluginImpl(LazyPlugin, plugin_type_cls): @@ -57,3 +66,30 @@ def get_plugin_cls(cls) -> PLUGIN_TYPES: return plugin_cls return LazyPluginImpl + + +def extra_deps(*extra_dep_names: Sequence[str]): + """Decorator to assign extra deps for the plugin class. + + There exist some plugins that cannot be executable with the default installation setup. + For example, `AcLauncher` plugin needs `tensorflow` and `openvino.tools` extra dependencies. + In this case, you have to add this decorator to that plugin class definition as follows. + + @extra_deps("tensorflow", "openvino.tools") + class AcLauncher(Launcher, CliPlugin): + ... + """ + + def inner(plugin_cls: object): + if hasattr(plugin_cls, _EXTRA_DEPS_ATTR_NAME): + raise DatumaroError(f"It already has {_EXTRA_DEPS_ATTR_NAME} attribute!") + + setattr(plugin_cls, _EXTRA_DEPS_ATTR_NAME, list(extra_dep_names)) + return plugin_cls + + return inner + + +def get_extra_deps(plugin_cls: object) -> List[str]: + """Get extra deps of the given plugin class. If not exists, return an empty list.""" + return getattr(plugin_cls, _EXTRA_DEPS_ATTR_NAME, []) diff --git a/datumaro/plugins/README.md b/datumaro/plugins/README.md index dc4d182ea1..7a4311bd44 100644 --- a/datumaro/plugins/README.md +++ b/datumaro/plugins/README.md @@ -1,5 +1,6 @@ -# Add a new builtin plugin +# How to develop a new builtin plugin +## Enroll a new builtin plugin If you are adding a new builtin plugin under `./datumaro/plugins/`, you have to enroll it to `./datumaro/plugins/specs.json` for the lazy import. Otherwise, you will be failed in this test (`tests/unit/test_environment.py`). You can enroll your plugin to `./datumaro/plugins/specs.json` with Python executable script as follows. @@ -9,3 +10,20 @@ python datumaro/plugins/specs.py ``` > **_NOTE:_** It is not recommended to mannually modify `./datumaro/plugins/specs.json` by hands. Please use the above Python executable script. + +## (Additional) Specify extra dependencies to plugin class + +If you are trying to add a new builtin plugin that has dependencies which is not installed together by default when installing datumaro. +You need to add Python decorator to specify extra deps for the plugin class definition. +This is required during the built-in plugin registration step to determine if a plugin is available by checking if its dependencies are installed on the system. + +For example, `AcLauncher` plugin needs `tensorflow` and `openvino.tools` extra dependencies. +Therefore, it added `@extra_deps("tensorflow", "openvino.tools")` to its class definition as follows. + +```python +from datumaro.components.lazy_plugin import extra_deps + +@extra_deps("tensorflow", "openvino.tools") +class AcLauncher(Launcher, CliPlugin): + ... +``` diff --git a/datumaro/plugins/accuracy_checker_plugin/ac_launcher.py b/datumaro/plugins/accuracy_checker_plugin/ac_launcher.py index 4ae32bbcf7..6d4bc97377 100644 --- a/datumaro/plugins/accuracy_checker_plugin/ac_launcher.py +++ b/datumaro/plugins/accuracy_checker_plugin/ac_launcher.py @@ -8,10 +8,12 @@ from datumaro.components.cli_plugin import CliPlugin from datumaro.components.launcher import Launcher +from datumaro.components.lazy_plugin import extra_deps from .details.ac import GenericAcLauncher as _GenericAcLauncher +@extra_deps("openvino.tools", "tensorflow") class AcLauncher(Launcher, CliPlugin): """ Generic model launcher with Accuracy Checker backend. diff --git a/datumaro/plugins/data_formats/tf_detection_api/base.py b/datumaro/plugins/data_formats/tf_detection_api/base.py index 3409796013..ae48997e1e 100644 --- a/datumaro/plugins/data_formats/tf_detection_api/base.py +++ b/datumaro/plugins/data_formats/tf_detection_api/base.py @@ -12,6 +12,7 @@ from datumaro.components.annotation import AnnotationType, Bbox, LabelCategories, Mask from datumaro.components.dataset_base import DatasetItem, SubsetBase from datumaro.components.importer import ImportContext, Importer +from datumaro.components.lazy_plugin import extra_deps from datumaro.components.media import Image from datumaro.util.image import decode_image, lazy_image from datumaro.util.tf_util import import_tf as _import_tf @@ -25,6 +26,7 @@ def clamp(value, _min, _max): return max(min(_max, value), _min) +@extra_deps("tensorflow") class TfDetectionApiBase(SubsetBase): def __init__( self, @@ -185,6 +187,7 @@ def _parse_tfrecord_file(cls, filepath, subset, images_dir): return dataset_items, dataset_labels +@extra_deps("tensorflow") class TfDetectionApiImporter(Importer): @classmethod def find_sources(cls, path): diff --git a/datumaro/plugins/data_formats/tf_detection_api/exporter.py b/datumaro/plugins/data_formats/tf_detection_api/exporter.py index f0383bdf7b..fa48f49bff 100644 --- a/datumaro/plugins/data_formats/tf_detection_api/exporter.py +++ b/datumaro/plugins/data_formats/tf_detection_api/exporter.py @@ -13,6 +13,7 @@ from datumaro.components.annotation import AnnotationType, LabelCategories from datumaro.components.errors import DatasetExportError from datumaro.components.exporter import Exporter +from datumaro.components.lazy_plugin import extra_deps from datumaro.components.media import ByteImage, Image, ImageFromBytes from datumaro.util.annotation_util import find_group_leader, find_instances, max_bbox from datumaro.util.image import encode_image @@ -52,6 +53,7 @@ def float_list_feature(value): return tf.train.Feature(float_list=tf.train.FloatList(value=value)) +@extra_deps("tensorflow") class TfDetectionApiExporter(Exporter): DEFAULT_IMAGE_EXT = DetectionApiPath.DEFAULT_IMAGE_EXT diff --git a/datumaro/plugins/specs.json b/datumaro/plugins/specs.json index 24430036f4..8f1f1c888f 100644 --- a/datumaro/plugins/specs.json +++ b/datumaro/plugins/specs.json @@ -1 +1 @@ -[{"import_path":"datumaro.plugins.transforms.AnnsToLabels","plugin_name":"anns_to_labels","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.BboxValuesDecrement","plugin_name":"bbox_values_decrement","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.BoxesToMasks","plugin_name":"boxes_to_masks","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.Correct","plugin_name":"correct","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.CropCoveredSegments","plugin_name":"crop_covered_segments","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.IdFromImageName","plugin_name":"id_from_image_name","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.MapSubsets","plugin_name":"map_subsets","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.MasksToPolygons","plugin_name":"masks_to_polygons","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.MergeInstanceSegments","plugin_name":"merge_instance_segments","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.PolygonsToMasks","plugin_name":"polygons_to_masks","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ProjectInfos","plugin_name":"project_infos","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ProjectLabels","plugin_name":"project_labels","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RandomSplit","plugin_name":"random_split","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.Reindex","plugin_name":"reindex","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ReindexAnnotations","plugin_name":"reindex_annotations","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RemapLabels","plugin_name":"remap_labels","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RemoveAnnotations","plugin_name":"remove_annotations","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RemoveAttributes","plugin_name":"remove_attributes","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.RemoveItems","plugin_name":"remove_items","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.Rename","plugin_name":"rename","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ResizeTransform","plugin_name":"resize","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.ShapesToBoxes","plugin_name":"shapes_to_boxes","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.transforms.Sort","plugin_name":"sort","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.splitter.Split","plugin_name":"split","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.ndr.NDR","plugin_name":"ndr","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesBase","plugin_name":"cityscapes","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.ade20k2020.Ade20k2020Base","plugin_name":"ade20k2020","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidBase","plugin_name":"camvid","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirBase","plugin_name":"image_dir","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.lfw.LfwBase","plugin_name":"lfw","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.vott_json.VottJsonBase","plugin_name":"vott_json","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.common_super_resolution.CommonSuperResolutionBase","plugin_name":"common_super_resolution","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.kinetics.KineticsBase","plugin_name":"kinetics","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtBase","plugin_name":"imagenet_txt","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mots.MotsPngExtractor","plugin_name":"mots_png_extractor","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.ade20k2017.Ade20k2017Base","plugin_name":"ade20k2017","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.nyu_depth_v2.NyuDepthV2Base","plugin_name":"nyu_depth_v2","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mnist.MnistBase","plugin_name":"mnist","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetBase","plugin_name":"imagenet","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceBase","plugin_name":"wider_face","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.brats.BratsBase","plugin_name":"brats","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Base","plugin_name":"vgg_face2","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.video.VideoFramesBase","plugin_name":"video_frames","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.video.VideoKeyframesBase","plugin_name":"video_keyframes","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mars.MarsBase","plugin_name":"mars","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.brats_numpy.BratsNumpyBase","plugin_name":"brats_numpy","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipBase","plugin_name":"image_zip","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.cifar.CifarBase","plugin_name":"cifar","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationBase","plugin_name":"common_semantic_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqBase","plugin_name":"mot_seq","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeBase","plugin_name":"label_me","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvBase","plugin_name":"mnist_csv","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Base","plugin_name":"market1501","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesBase","plugin_name":"open_images","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.vott_csv.VottCsvBase","plugin_name":"vott_csv","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.tiling.tile.Tile","plugin_name":"tile","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.tiling.merge_tile.MergeTile","plugin_name":"merge_tile","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.sampler.random_sampler.LabelRandomSampler","plugin_name":"label_random_sampler","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.sampler.random_sampler.RandomSampler","plugin_name":"random_sampler","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.sampler.relevancy_sampler.RelevancySampler","plugin_name":"relevancy_sampler","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.celeba.celeba.CelebaBase","plugin_name":"celeba","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.celeba.align_celeba.AlignCelebaBase","plugin_name":"align_celeba","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.segment_anything.base.SegmentAnythingBase","plugin_name":"segment_anything","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.base.SuperviselyPointCloudBase","plugin_name":"sly_pointcloud","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextLocalizationBase","plugin_name":"icdar_text_localization","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextSegmentationBase","plugin_name":"icdar_text_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarWordRecognitionBase","plugin_name":"icdar_word_recognition","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloLooseBase","plugin_name":"yolo_loose","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloStrictBase","plugin_name":"yolo_strict","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloUltralyticsBase","plugin_name":"yolo_ultralytics","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_mat.MpiiBase","plugin_name":"mpii","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_json.MpiiJsonBase","plugin_name":"mpii_json","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaBase","plugin_name":"ava","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocActionBase","plugin_name":"voc_action","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocBase","plugin_name":"voc","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocClassificationBase","plugin_name":"voc_classification","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocDetectionBase","plugin_name":"voc_detection","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocInstanceSegmentationBase","plugin_name":"voc_instance_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocLayoutBase","plugin_name":"voc_layout","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.voc.base.VocSegmentationBase","plugin_name":"voc_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoCaptionsBase","plugin_name":"coco_captions","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoImageInfoBase","plugin_name":"coco_image_info","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoInstancesBase","plugin_name":"coco_instances","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoLabelsBase","plugin_name":"coco_labels","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoPanopticBase","plugin_name":"coco_panoptic","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoPersonKeypointsBase","plugin_name":"coco_person_keypoints","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoStuffBase","plugin_name":"coco_stuff","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.arrow.base.ArrowBase","plugin_name":"arrow","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.datumaro.base.DatumaroBase","plugin_name":"datumaro","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaAlBase","plugin_name":"synthia_al","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaRandBase","plugin_name":"synthia_rand","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaSfBase","plugin_name":"synthia_sf","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecClassificationBase","plugin_name":"mvtec_classification","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecDetectionBase","plugin_name":"mvtec_detection","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecSegmentationBase","plugin_name":"mvtec_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.cvat.base.CvatBase","plugin_name":"cvat","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.kitti_raw.base.KittiRawBase","plugin_name":"kitti_raw","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.base.TfDetectionApiBase","plugin_name":"tf_detection_api","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.kitti.base.KittiDetectionBase","plugin_name":"kitti_detection","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.kitti.base.KittiSegmentationBase","plugin_name":"kitti_segmentation","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.base.DatumaroBinaryBase","plugin_name":"datumaro_binary","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.base.MapillaryVistasInstancesBase","plugin_name":"mapillary_vistas_instances","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.base.MapillaryVistasPanopticBase","plugin_name":"mapillary_vistas_panoptic","plugin_type":"DatasetBase"},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesImporter","plugin_name":"cityscapes","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.ade20k2020.Ade20k2020Importer","plugin_name":"ade20k2020","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidImporter","plugin_name":"camvid","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirImporter","plugin_name":"image_dir","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.lfw.LfwImporter","plugin_name":"lfw","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.vott_json.VottJsonImporter","plugin_name":"vott_json","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.common_super_resolution.CommonSuperResolutionImporter","plugin_name":"common_super_resolution","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kinetics.KineticsImporter","plugin_name":"kinetics","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtImporter","plugin_name":"imagenet_txt","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mots.MotsImporter","plugin_name":"mots","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.ade20k2017.Ade20k2017Importer","plugin_name":"ade20k2017","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.nyu_depth_v2.NyuDepthV2Importer","plugin_name":"nyu_depth_v2","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mnist.MnistImporter","plugin_name":"mnist","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetImporter","plugin_name":"imagenet","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetWithSubsetDirsImporter","plugin_name":"imagenet_with_subset_dirs","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceImporter","plugin_name":"wider_face","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.brats.BratsImporter","plugin_name":"brats","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Importer","plugin_name":"vgg_face2","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.video.VideoFramesImporter","plugin_name":"video_frames","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.video.VideoKeyframesImporter","plugin_name":"video_keyframes","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mars.MarsImporter","plugin_name":"mars","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.brats_numpy.BratsNumpyImporter","plugin_name":"brats_numpy","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipImporter","plugin_name":"image_zip","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.cifar.CifarImporter","plugin_name":"cifar","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationImporter","plugin_name":"common_semantic_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationWithSubsetDirsImporter","plugin_name":"common_semantic_segmentation_with_subset_dirs","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqImporter","plugin_name":"mot_seq","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeImporter","plugin_name":"label_me","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvImporter","plugin_name":"mnist_csv","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Importer","plugin_name":"market1501","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesImporter","plugin_name":"open_images","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.vott_csv.VottCsvImporter","plugin_name":"vott_csv","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.celeba.celeba.CelebaImporter","plugin_name":"celeba","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.celeba.align_celeba.AlignCelebaImporter","plugin_name":"align_celeba","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.segment_anything.importer.SegmentAnythingImporter","plugin_name":"segment_anything","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.base.SuperviselyPointCloudImporter","plugin_name":"sly_pointcloud","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextLocalizationImporter","plugin_name":"icdar_text_localization","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextSegmentationImporter","plugin_name":"icdar_text_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarWordRecognitionImporter","plugin_name":"icdar_word_recognition","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.yolo.importer.YoloImporter","plugin_name":"yolo","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_mat.MpiiImporter","plugin_name":"mpii","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_json.MpiiJsonImporter","plugin_name":"mpii_json","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaImporter","plugin_name":"ava","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocActionImporter","plugin_name":"voc_action","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocClassificationImporter","plugin_name":"voc_classification","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocDetectionImporter","plugin_name":"voc_detection","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocImporter","plugin_name":"voc","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocInstanceSegmentationImporter","plugin_name":"voc_instance_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocLayoutImporter","plugin_name":"voc_layout","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocSegmentationImporter","plugin_name":"voc_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoCaptionsImporter","plugin_name":"coco_captions","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoImageInfoImporter","plugin_name":"coco_image_info","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoImporter","plugin_name":"coco","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoInstancesImporter","plugin_name":"coco_instances","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoLabelsImporter","plugin_name":"coco_labels","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoPanopticImporter","plugin_name":"coco_panoptic","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoPersonKeypointsImporter","plugin_name":"coco_person_keypoints","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoRoboflowImporter","plugin_name":"coco_roboflow","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoStuffImporter","plugin_name":"coco_stuff","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.arrow.importer.ArrowImporter","plugin_name":"arrow","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaAlImporter","plugin_name":"synthia_al","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaRandImporter","plugin_name":"synthia_rand","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaSfImporter","plugin_name":"synthia_sf","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecClassificationImporter","plugin_name":"mvtec_classification","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecDetectionImporter","plugin_name":"mvtec_detection","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecImporter","plugin_name":"mvtec","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecSegmentationImporter","plugin_name":"mvtec_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.cvat.base.CvatImporter","plugin_name":"cvat","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kitti_raw.base.KittiRawImporter","plugin_name":"kitti_raw","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.base.TfDetectionApiImporter","plugin_name":"tf_detection_api","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.datumaro.importer.DatumaroImporter","plugin_name":"datumaro","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiDetectionImporter","plugin_name":"kitti_detection","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiImporter","plugin_name":"kitti","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiSegmentationImporter","plugin_name":"kitti_segmentation","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.importer.DatumaroBinaryImporter","plugin_name":"datumaro_binary","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasImporter","plugin_name":"mapillary_vistas","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasInstancesImporter","plugin_name":"mapillary_vistas_instances","plugin_type":"Importer"},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasPanopticImporter","plugin_name":"mapillary_vistas_panoptic","plugin_type":"Importer"},{"import_path":"datumaro.plugins.explorer.ExplorerLauncher","plugin_name":"explorer","plugin_type":"Launcher"},{"import_path":"datumaro.plugins.openvino_plugin.launcher.OpenvinoLauncher","plugin_name":"openvino","plugin_type":"Launcher"},{"import_path":"datumaro.plugins.shift_analyzer.ShiftAnalyzerLauncher","plugin_name":"shift_analyzer","plugin_type":"Launcher"},{"import_path":"datumaro.plugins.accuracy_checker_plugin.ac_launcher.AcLauncher","plugin_name":"ac","plugin_type":"Launcher"},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesExporter","plugin_name":"cityscapes","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidExporter","plugin_name":"camvid","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirExporter","plugin_name":"image_dir","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.lfw.LfwExporter","plugin_name":"lfw","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtExporter","plugin_name":"imagenet_txt","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mots.MotsPngExporter","plugin_name":"mots_png","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mnist.MnistExporter","plugin_name":"mnist","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetExporter","plugin_name":"imagenet","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetWithSubsetDirsExporter","plugin_name":"imagenet_with_subset_dirs","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceExporter","plugin_name":"wider_face","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Exporter","plugin_name":"vgg_face2","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipExporter","plugin_name":"image_zip","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.cifar.CifarExporter","plugin_name":"cifar","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqGtExporter","plugin_name":"mot_seq_gt","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeExporter","plugin_name":"label_me","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvExporter","plugin_name":"mnist_csv","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Exporter","plugin_name":"market1501","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesExporter","plugin_name":"open_images","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.segment_anything.exporter.SegmentAnythingExporter","plugin_name":"segment_anything","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.exporter.SuperviselyPointCloudExporter","plugin_name":"sly_pointcloud","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarTextLocalizationExporter","plugin_name":"icdar_text_localization","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarTextSegmentationExporter","plugin_name":"icdar_text_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarWordRecognitionExporter","plugin_name":"icdar_word_recognition","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.yolo.exporter.YoloExporter","plugin_name":"yolo","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.yolo.exporter.YoloUltralyticsExporter","plugin_name":"yolo_ultralytics","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaExporter","plugin_name":"ava","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocActionExporter","plugin_name":"voc_action","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocClassificationExporter","plugin_name":"voc_classification","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocDetectionExporter","plugin_name":"voc_detection","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocExporter","plugin_name":"voc","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocInstanceSegmentationExporter","plugin_name":"voc_instance_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocLayoutExporter","plugin_name":"voc_layout","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocSegmentationExporter","plugin_name":"voc_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoCaptionsExporter","plugin_name":"coco_captions","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoExporter","plugin_name":"coco","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoImageInfoExporter","plugin_name":"coco_image_info","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoInstancesExporter","plugin_name":"coco_instances","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoLabelsExporter","plugin_name":"coco_labels","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoPanopticExporter","plugin_name":"coco_panoptic","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoPersonKeypointsExporter","plugin_name":"coco_person_keypoints","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoStuffExporter","plugin_name":"coco_stuff","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.arrow.exporter.ArrowExporter","plugin_name":"arrow","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecClassificationExporter","plugin_name":"mvtec_classification","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecDetectionExporter","plugin_name":"mvtec_detection","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecExporter","plugin_name":"mvtec","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecSegmentationExporter","plugin_name":"mvtec_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.cvat.exporter.CvatExporter","plugin_name":"cvat","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.kitti_raw.exporter.KittiRawExporter","plugin_name":"kitti_raw","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.exporter.TfDetectionApiExporter","plugin_name":"tf_detection_api","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.datumaro.exporter.DatumaroExporter","plugin_name":"datumaro","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiDetectionExporter","plugin_name":"kitti_detection","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiExporter","plugin_name":"kitti","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiSegmentationExporter","plugin_name":"kitti_segmentation","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.exporter.DatumaroBinaryExporter","plugin_name":"datumaro_binary","plugin_type":"Exporter"},{"import_path":"datumaro.plugins.transforms.AnnsToLabels","plugin_name":"anns_to_labels","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.BboxValuesDecrement","plugin_name":"bbox_values_decrement","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.BoxesToMasks","plugin_name":"boxes_to_masks","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.Correct","plugin_name":"correct","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.CropCoveredSegments","plugin_name":"crop_covered_segments","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.IdFromImageName","plugin_name":"id_from_image_name","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.MapSubsets","plugin_name":"map_subsets","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.MasksToPolygons","plugin_name":"masks_to_polygons","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.MergeInstanceSegments","plugin_name":"merge_instance_segments","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.PolygonsToMasks","plugin_name":"polygons_to_masks","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ProjectInfos","plugin_name":"project_infos","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ProjectLabels","plugin_name":"project_labels","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RandomSplit","plugin_name":"random_split","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.Reindex","plugin_name":"reindex","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ReindexAnnotations","plugin_name":"reindex_annotations","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RemapLabels","plugin_name":"remap_labels","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RemoveAnnotations","plugin_name":"remove_annotations","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RemoveAttributes","plugin_name":"remove_attributes","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.RemoveItems","plugin_name":"remove_items","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.Rename","plugin_name":"rename","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ResizeTransform","plugin_name":"resize","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.ShapesToBoxes","plugin_name":"shapes_to_boxes","plugin_type":"Transform"},{"import_path":"datumaro.plugins.transforms.Sort","plugin_name":"sort","plugin_type":"Transform"},{"import_path":"datumaro.plugins.splitter.Split","plugin_name":"split","plugin_type":"Transform"},{"import_path":"datumaro.plugins.ndr.NDR","plugin_name":"ndr","plugin_type":"Transform"},{"import_path":"datumaro.plugins.tiling.tile.Tile","plugin_name":"tile","plugin_type":"Transform"},{"import_path":"datumaro.plugins.tiling.merge_tile.MergeTile","plugin_name":"merge_tile","plugin_type":"Transform"},{"import_path":"datumaro.plugins.sampler.random_sampler.LabelRandomSampler","plugin_name":"label_random_sampler","plugin_type":"Transform"},{"import_path":"datumaro.plugins.sampler.random_sampler.RandomSampler","plugin_name":"random_sampler","plugin_type":"Transform"},{"import_path":"datumaro.plugins.sampler.relevancy_sampler.RelevancySampler","plugin_name":"relevancy_sampler","plugin_type":"Transform"},{"import_path":"datumaro.plugins.validators.ClassificationValidator","plugin_name":"classification","plugin_type":"Validator"},{"import_path":"datumaro.plugins.validators.DetectionValidator","plugin_name":"detection","plugin_type":"Validator"},{"import_path":"datumaro.plugins.validators.SegmentationValidator","plugin_name":"segmentation","plugin_type":"Validator"}] +[{"import_path":"datumaro.plugins.transforms.AnnsToLabels","plugin_name":"anns_to_labels","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.BboxValuesDecrement","plugin_name":"bbox_values_decrement","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.BoxesToMasks","plugin_name":"boxes_to_masks","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.Correct","plugin_name":"correct","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.CropCoveredSegments","plugin_name":"crop_covered_segments","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.IdFromImageName","plugin_name":"id_from_image_name","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.MapSubsets","plugin_name":"map_subsets","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.MasksToPolygons","plugin_name":"masks_to_polygons","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.MergeInstanceSegments","plugin_name":"merge_instance_segments","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.PolygonsToMasks","plugin_name":"polygons_to_masks","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ProjectInfos","plugin_name":"project_infos","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ProjectLabels","plugin_name":"project_labels","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RandomSplit","plugin_name":"random_split","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.Reindex","plugin_name":"reindex","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ReindexAnnotations","plugin_name":"reindex_annotations","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RemapLabels","plugin_name":"remap_labels","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RemoveAnnotations","plugin_name":"remove_annotations","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RemoveAttributes","plugin_name":"remove_attributes","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RemoveItems","plugin_name":"remove_items","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.Rename","plugin_name":"rename","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ResizeTransform","plugin_name":"resize","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ShapesToBoxes","plugin_name":"shapes_to_boxes","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.Sort","plugin_name":"sort","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.splitter.Split","plugin_name":"split","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.ndr.NDR","plugin_name":"ndr","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesBase","plugin_name":"cityscapes","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.ade20k2020.Ade20k2020Base","plugin_name":"ade20k2020","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidBase","plugin_name":"camvid","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirBase","plugin_name":"image_dir","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.lfw.LfwBase","plugin_name":"lfw","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.vott_json.VottJsonBase","plugin_name":"vott_json","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.common_super_resolution.CommonSuperResolutionBase","plugin_name":"common_super_resolution","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kinetics.KineticsBase","plugin_name":"kinetics","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtBase","plugin_name":"imagenet_txt","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mots.MotsPngExtractor","plugin_name":"mots_png_extractor","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.ade20k2017.Ade20k2017Base","plugin_name":"ade20k2017","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.nyu_depth_v2.NyuDepthV2Base","plugin_name":"nyu_depth_v2","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mnist.MnistBase","plugin_name":"mnist","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetBase","plugin_name":"imagenet","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceBase","plugin_name":"wider_face","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.brats.BratsBase","plugin_name":"brats","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Base","plugin_name":"vgg_face2","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.video.VideoFramesBase","plugin_name":"video_frames","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.video.VideoKeyframesBase","plugin_name":"video_keyframes","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mars.MarsBase","plugin_name":"mars","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.brats_numpy.BratsNumpyBase","plugin_name":"brats_numpy","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipBase","plugin_name":"image_zip","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.cifar.CifarBase","plugin_name":"cifar","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationBase","plugin_name":"common_semantic_segmentation","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqBase","plugin_name":"mot_seq","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeBase","plugin_name":"label_me","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvBase","plugin_name":"mnist_csv","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Base","plugin_name":"market1501","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesBase","plugin_name":"open_images","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.vott_csv.VottCsvBase","plugin_name":"vott_csv","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.tiling.tile.Tile","plugin_name":"tile","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.tiling.merge_tile.MergeTile","plugin_name":"merge_tile","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.sampler.random_sampler.LabelRandomSampler","plugin_name":"label_random_sampler","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.sampler.random_sampler.RandomSampler","plugin_name":"random_sampler","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.sampler.relevancy_sampler.RelevancySampler","plugin_name":"relevancy_sampler","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.celeba.celeba.CelebaBase","plugin_name":"celeba","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.celeba.align_celeba.AlignCelebaBase","plugin_name":"align_celeba","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.segment_anything.base.SegmentAnythingBase","plugin_name":"segment_anything","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.base.SuperviselyPointCloudBase","plugin_name":"sly_pointcloud","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextLocalizationBase","plugin_name":"icdar_text_localization","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextSegmentationBase","plugin_name":"icdar_text_segmentation","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarWordRecognitionBase","plugin_name":"icdar_word_recognition","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloLooseBase","plugin_name":"yolo_loose","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloStrictBase","plugin_name":"yolo_strict","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.yolo.base.YoloUltralyticsBase","plugin_name":"yolo_ultralytics","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_mat.MpiiBase","plugin_name":"mpii","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_json.MpiiJsonBase","plugin_name":"mpii_json","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaBase","plugin_name":"ava","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.base.VocActionBase","plugin_name":"voc_action","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.base.VocBase","plugin_name":"voc","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.base.VocClassificationBase","plugin_name":"voc_classification","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.base.VocDetectionBase","plugin_name":"voc_detection","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.base.VocInstanceSegmentationBase","plugin_name":"voc_instance_segmentation","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.base.VocLayoutBase","plugin_name":"voc_layout","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.base.VocSegmentationBase","plugin_name":"voc_segmentation","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoCaptionsBase","plugin_name":"coco_captions","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoImageInfoBase","plugin_name":"coco_image_info","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoInstancesBase","plugin_name":"coco_instances","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoLabelsBase","plugin_name":"coco_labels","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoPanopticBase","plugin_name":"coco_panoptic","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoPersonKeypointsBase","plugin_name":"coco_person_keypoints","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.base.CocoStuffBase","plugin_name":"coco_stuff","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.arrow.base.ArrowBase","plugin_name":"arrow","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.datumaro.base.DatumaroBase","plugin_name":"datumaro","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaAlBase","plugin_name":"synthia_al","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaRandBase","plugin_name":"synthia_rand","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.synthia.base.SynthiaSfBase","plugin_name":"synthia_sf","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecClassificationBase","plugin_name":"mvtec_classification","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecDetectionBase","plugin_name":"mvtec_detection","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.base.MvtecSegmentationBase","plugin_name":"mvtec_segmentation","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.cvat.base.CvatBase","plugin_name":"cvat","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti_raw.base.KittiRawBase","plugin_name":"kitti_raw","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.base.TfDetectionApiBase","plugin_name":"tf_detection_api","plugin_type":"DatasetBase","extra_deps":["tensorflow"]},{"import_path":"datumaro.plugins.data_formats.kitti.base.KittiDetectionBase","plugin_name":"kitti_detection","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti.base.KittiSegmentationBase","plugin_name":"kitti_segmentation","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.base.DatumaroBinaryBase","plugin_name":"datumaro_binary","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.base.MapillaryVistasInstancesBase","plugin_name":"mapillary_vistas_instances","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.base.MapillaryVistasPanopticBase","plugin_name":"mapillary_vistas_panoptic","plugin_type":"DatasetBase","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesImporter","plugin_name":"cityscapes","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.ade20k2020.Ade20k2020Importer","plugin_name":"ade20k2020","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidImporter","plugin_name":"camvid","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirImporter","plugin_name":"image_dir","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.lfw.LfwImporter","plugin_name":"lfw","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.vott_json.VottJsonImporter","plugin_name":"vott_json","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.common_super_resolution.CommonSuperResolutionImporter","plugin_name":"common_super_resolution","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kinetics.KineticsImporter","plugin_name":"kinetics","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtImporter","plugin_name":"imagenet_txt","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mots.MotsImporter","plugin_name":"mots","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.ade20k2017.Ade20k2017Importer","plugin_name":"ade20k2017","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.nyu_depth_v2.NyuDepthV2Importer","plugin_name":"nyu_depth_v2","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mnist.MnistImporter","plugin_name":"mnist","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetImporter","plugin_name":"imagenet","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetWithSubsetDirsImporter","plugin_name":"imagenet_with_subset_dirs","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceImporter","plugin_name":"wider_face","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.brats.BratsImporter","plugin_name":"brats","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Importer","plugin_name":"vgg_face2","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.video.VideoFramesImporter","plugin_name":"video_frames","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.video.VideoKeyframesImporter","plugin_name":"video_keyframes","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mars.MarsImporter","plugin_name":"mars","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.brats_numpy.BratsNumpyImporter","plugin_name":"brats_numpy","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipImporter","plugin_name":"image_zip","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.cifar.CifarImporter","plugin_name":"cifar","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationImporter","plugin_name":"common_semantic_segmentation","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.common_semantic_segmentation.CommonSemanticSegmentationWithSubsetDirsImporter","plugin_name":"common_semantic_segmentation_with_subset_dirs","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqImporter","plugin_name":"mot_seq","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeImporter","plugin_name":"label_me","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvImporter","plugin_name":"mnist_csv","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Importer","plugin_name":"market1501","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesImporter","plugin_name":"open_images","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.vott_csv.VottCsvImporter","plugin_name":"vott_csv","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.celeba.celeba.CelebaImporter","plugin_name":"celeba","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.celeba.align_celeba.AlignCelebaImporter","plugin_name":"align_celeba","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.segment_anything.importer.SegmentAnythingImporter","plugin_name":"segment_anything","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.base.SuperviselyPointCloudImporter","plugin_name":"sly_pointcloud","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextLocalizationImporter","plugin_name":"icdar_text_localization","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarTextSegmentationImporter","plugin_name":"icdar_text_segmentation","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.base.IcdarWordRecognitionImporter","plugin_name":"icdar_word_recognition","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.yolo.importer.YoloImporter","plugin_name":"yolo","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_mat.MpiiImporter","plugin_name":"mpii","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mpii.mpii_json.MpiiJsonImporter","plugin_name":"mpii_json","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaImporter","plugin_name":"ava","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocActionImporter","plugin_name":"voc_action","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocClassificationImporter","plugin_name":"voc_classification","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocDetectionImporter","plugin_name":"voc_detection","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocImporter","plugin_name":"voc","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocInstanceSegmentationImporter","plugin_name":"voc_instance_segmentation","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocLayoutImporter","plugin_name":"voc_layout","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.importer.VocSegmentationImporter","plugin_name":"voc_segmentation","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoCaptionsImporter","plugin_name":"coco_captions","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoImageInfoImporter","plugin_name":"coco_image_info","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoImporter","plugin_name":"coco","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoInstancesImporter","plugin_name":"coco_instances","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoLabelsImporter","plugin_name":"coco_labels","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoPanopticImporter","plugin_name":"coco_panoptic","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoPersonKeypointsImporter","plugin_name":"coco_person_keypoints","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoRoboflowImporter","plugin_name":"coco_roboflow","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.importer.CocoStuffImporter","plugin_name":"coco_stuff","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.arrow.importer.ArrowImporter","plugin_name":"arrow","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaAlImporter","plugin_name":"synthia_al","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaRandImporter","plugin_name":"synthia_rand","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.synthia.importer.SynthiaSfImporter","plugin_name":"synthia_sf","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecClassificationImporter","plugin_name":"mvtec_classification","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecDetectionImporter","plugin_name":"mvtec_detection","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecImporter","plugin_name":"mvtec","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.importer.MvtecSegmentationImporter","plugin_name":"mvtec_segmentation","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.cvat.base.CvatImporter","plugin_name":"cvat","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti_raw.base.KittiRawImporter","plugin_name":"kitti_raw","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.base.TfDetectionApiImporter","plugin_name":"tf_detection_api","plugin_type":"Importer","extra_deps":["tensorflow"]},{"import_path":"datumaro.plugins.data_formats.datumaro.importer.DatumaroImporter","plugin_name":"datumaro","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiDetectionImporter","plugin_name":"kitti_detection","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiImporter","plugin_name":"kitti","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti.importer.KittiSegmentationImporter","plugin_name":"kitti_segmentation","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.importer.DatumaroBinaryImporter","plugin_name":"datumaro_binary","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasImporter","plugin_name":"mapillary_vistas","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasInstancesImporter","plugin_name":"mapillary_vistas_instances","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mapillary_vistas.importer.MapillaryVistasPanopticImporter","plugin_name":"mapillary_vistas_panoptic","plugin_type":"Importer","extra_deps":[]},{"import_path":"datumaro.plugins.explorer.ExplorerLauncher","plugin_name":"explorer","plugin_type":"Launcher","extra_deps":[]},{"import_path":"datumaro.plugins.openvino_plugin.launcher.OpenvinoLauncher","plugin_name":"openvino","plugin_type":"Launcher","extra_deps":[]},{"import_path":"datumaro.plugins.shift_analyzer.ShiftAnalyzerLauncher","plugin_name":"shift_analyzer","plugin_type":"Launcher","extra_deps":[]},{"import_path":"datumaro.plugins.accuracy_checker_plugin.ac_launcher.AcLauncher","plugin_name":"ac","plugin_type":"Launcher","extra_deps":["openvino.tools","tensorflow"]},{"import_path":"datumaro.plugins.data_formats.cityscapes.CityscapesExporter","plugin_name":"cityscapes","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.camvid.CamvidExporter","plugin_name":"camvid","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.image_dir.ImageDirExporter","plugin_name":"image_dir","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.lfw.LfwExporter","plugin_name":"lfw","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.imagenet_txt.ImagenetTxtExporter","plugin_name":"imagenet_txt","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mots.MotsPngExporter","plugin_name":"mots_png","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mnist.MnistExporter","plugin_name":"mnist","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetExporter","plugin_name":"imagenet","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.imagenet.ImagenetWithSubsetDirsExporter","plugin_name":"imagenet_with_subset_dirs","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.widerface.WiderFaceExporter","plugin_name":"wider_face","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.vgg_face2.VggFace2Exporter","plugin_name":"vgg_face2","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.image_zip.ImageZipExporter","plugin_name":"image_zip","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.cifar.CifarExporter","plugin_name":"cifar","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mot.MotSeqGtExporter","plugin_name":"mot_seq_gt","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.labelme.LabelMeExporter","plugin_name":"label_me","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mnist_csv.MnistCsvExporter","plugin_name":"mnist_csv","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.market1501.Market1501Exporter","plugin_name":"market1501","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.open_images.OpenImagesExporter","plugin_name":"open_images","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.segment_anything.exporter.SegmentAnythingExporter","plugin_name":"segment_anything","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.sly_pointcloud.exporter.SuperviselyPointCloudExporter","plugin_name":"sly_pointcloud","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarTextLocalizationExporter","plugin_name":"icdar_text_localization","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarTextSegmentationExporter","plugin_name":"icdar_text_segmentation","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.icdar.exporter.IcdarWordRecognitionExporter","plugin_name":"icdar_word_recognition","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.yolo.exporter.YoloExporter","plugin_name":"yolo","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.yolo.exporter.YoloUltralyticsExporter","plugin_name":"yolo_ultralytics","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.ava.ava.AvaExporter","plugin_name":"ava","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocActionExporter","plugin_name":"voc_action","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocClassificationExporter","plugin_name":"voc_classification","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocDetectionExporter","plugin_name":"voc_detection","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocExporter","plugin_name":"voc","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocInstanceSegmentationExporter","plugin_name":"voc_instance_segmentation","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocLayoutExporter","plugin_name":"voc_layout","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.voc.exporter.VocSegmentationExporter","plugin_name":"voc_segmentation","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoCaptionsExporter","plugin_name":"coco_captions","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoExporter","plugin_name":"coco","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoImageInfoExporter","plugin_name":"coco_image_info","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoInstancesExporter","plugin_name":"coco_instances","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoLabelsExporter","plugin_name":"coco_labels","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoPanopticExporter","plugin_name":"coco_panoptic","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoPersonKeypointsExporter","plugin_name":"coco_person_keypoints","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.coco.exporter.CocoStuffExporter","plugin_name":"coco_stuff","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.arrow.exporter.ArrowExporter","plugin_name":"arrow","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecClassificationExporter","plugin_name":"mvtec_classification","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecDetectionExporter","plugin_name":"mvtec_detection","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecExporter","plugin_name":"mvtec","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.mvtec.exporter.MvtecSegmentationExporter","plugin_name":"mvtec_segmentation","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.cvat.exporter.CvatExporter","plugin_name":"cvat","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti_raw.exporter.KittiRawExporter","plugin_name":"kitti_raw","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.tf_detection_api.exporter.TfDetectionApiExporter","plugin_name":"tf_detection_api","plugin_type":"Exporter","extra_deps":["tensorflow"]},{"import_path":"datumaro.plugins.data_formats.datumaro.exporter.DatumaroExporter","plugin_name":"datumaro","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiDetectionExporter","plugin_name":"kitti_detection","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiExporter","plugin_name":"kitti","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.kitti.exporter.KittiSegmentationExporter","plugin_name":"kitti_segmentation","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.data_formats.datumaro_binary.exporter.DatumaroBinaryExporter","plugin_name":"datumaro_binary","plugin_type":"Exporter","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.AnnsToLabels","plugin_name":"anns_to_labels","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.BboxValuesDecrement","plugin_name":"bbox_values_decrement","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.BoxesToMasks","plugin_name":"boxes_to_masks","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.Correct","plugin_name":"correct","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.CropCoveredSegments","plugin_name":"crop_covered_segments","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.IdFromImageName","plugin_name":"id_from_image_name","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.MapSubsets","plugin_name":"map_subsets","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.MasksToPolygons","plugin_name":"masks_to_polygons","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.MergeInstanceSegments","plugin_name":"merge_instance_segments","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.PolygonsToMasks","plugin_name":"polygons_to_masks","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ProjectInfos","plugin_name":"project_infos","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ProjectLabels","plugin_name":"project_labels","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RandomSplit","plugin_name":"random_split","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.Reindex","plugin_name":"reindex","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ReindexAnnotations","plugin_name":"reindex_annotations","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RemapLabels","plugin_name":"remap_labels","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RemoveAnnotations","plugin_name":"remove_annotations","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RemoveAttributes","plugin_name":"remove_attributes","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.RemoveItems","plugin_name":"remove_items","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.Rename","plugin_name":"rename","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ResizeTransform","plugin_name":"resize","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.ShapesToBoxes","plugin_name":"shapes_to_boxes","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.transforms.Sort","plugin_name":"sort","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.splitter.Split","plugin_name":"split","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.ndr.NDR","plugin_name":"ndr","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.tiling.tile.Tile","plugin_name":"tile","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.tiling.merge_tile.MergeTile","plugin_name":"merge_tile","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.sampler.random_sampler.LabelRandomSampler","plugin_name":"label_random_sampler","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.sampler.random_sampler.RandomSampler","plugin_name":"random_sampler","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.sampler.relevancy_sampler.RelevancySampler","plugin_name":"relevancy_sampler","plugin_type":"Transform","extra_deps":[]},{"import_path":"datumaro.plugins.validators.ClassificationValidator","plugin_name":"classification","plugin_type":"Validator","extra_deps":[]},{"import_path":"datumaro.plugins.validators.DetectionValidator","plugin_name":"detection","plugin_type":"Validator","extra_deps":[]},{"import_path":"datumaro.plugins.validators.SegmentationValidator","plugin_name":"segmentation","plugin_type":"Validator","extra_deps":[]}] diff --git a/datumaro/plugins/specs.py b/datumaro/plugins/specs.py index cb707e8e23..727047460d 100644 --- a/datumaro/plugins/specs.py +++ b/datumaro/plugins/specs.py @@ -4,7 +4,7 @@ from pathlib import Path -from datumaro.components.lazy_plugin import get_lazy_plugin +from datumaro.components.lazy_plugin import get_extra_deps, get_lazy_plugin from datumaro.util import parse_json_file _SOURCE_PATH = Path(__file__).resolve() @@ -12,8 +12,14 @@ _SPECS_JSON_PATH = _SOURCE_DIR / "specs.json" LAZY_PLUGINS = [ - get_lazy_plugin(spec["import_path"], spec["plugin_name"], spec["plugin_type"]) - for spec in parse_json_file(str(_SPECS_JSON_PATH)) + plugin + for plugin in [ + get_lazy_plugin( + spec["import_path"], spec["plugin_name"], spec["plugin_type"], spec["extra_deps"] + ) + for spec in parse_json_file(str(_SPECS_JSON_PATH)) + ] + if plugin is not None ] if __name__ == "__main__": @@ -35,6 +41,7 @@ def _enroll_to_plugin_specs(plugins, plugin_type): "import_path": f"{mod}.{class_name}", "plugin_name": plugin_name, "plugin_type": plugin_type, + "extra_deps": get_extra_deps(plugin), } ] diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py index 27f1ce353f..5466ca2e63 100644 --- a/tests/unit/test_environment.py +++ b/tests/unit/test_environment.py @@ -2,6 +2,8 @@ # # SPDX-License-Identifier: MIT +import sys + import pytest from datumaro.components.environment import Environment @@ -30,3 +32,28 @@ def test_equivalance(self, fxt_lazy_import: Environment, fxt_no_lazy_import: Env assert sorted(fxt_lazy_import.generators) == sorted(fxt_no_lazy_import.generators) assert sorted(fxt_lazy_import.transforms) == sorted(fxt_no_lazy_import.transforms) assert sorted(fxt_lazy_import.validators) == sorted(fxt_no_lazy_import.validators) + + @pytest.fixture + def fxt_tf_failure(self, monkeypatch): + # Simulate `tensorflow` import failure + monkeypatch.setitem(sys.modules, "tensorflow", None) + + with Environment.release_builtin_plugins(): + env = Environment(use_lazy_import=True) + _ = env.importers + yield env + + def test_extra_deps_req(self, fxt_tf_failure: Environment): + """Plugins affected by tensorflow import failure: `ac` and `tf_detection_api`.""" + loaded_plugin_names = set( + sorted(fxt_tf_failure.extractors) + + sorted(fxt_tf_failure.importers) + + sorted(fxt_tf_failure.launchers) + + sorted(fxt_tf_failure.exporters) + + sorted(fxt_tf_failure.generators) + + sorted(fxt_tf_failure.transforms) + + sorted(fxt_tf_failure.validators) + ) + + assert "ac" not in loaded_plugin_names + assert "tf_detection_api" not in loaded_plugin_names From 705f959dc8b6a13511679d82f41103a4129a448a Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Wed, 31 May 2023 16:26:08 +0900 Subject: [PATCH 08/10] Add openvino.tools import failure Signed-off-by: Kim, Vinnam --- tests/unit/test_environment.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py index 5466ca2e63..7a688f88a3 100644 --- a/tests/unit/test_environment.py +++ b/tests/unit/test_environment.py @@ -35,7 +35,8 @@ def test_equivalance(self, fxt_lazy_import: Environment, fxt_no_lazy_import: Env @pytest.fixture def fxt_tf_failure(self, monkeypatch): - # Simulate `tensorflow` import failure + # Simulate `openvino.tools` and `tensorflow` import failure + monkeypatch.setitem(sys.modules, "openvino.tools", None) monkeypatch.setitem(sys.modules, "tensorflow", None) with Environment.release_builtin_plugins(): @@ -44,7 +45,7 @@ def fxt_tf_failure(self, monkeypatch): yield env def test_extra_deps_req(self, fxt_tf_failure: Environment): - """Plugins affected by tensorflow import failure: `ac` and `tf_detection_api`.""" + """Plugins affected by the import failure: `ac` and `tf_detection_api`.""" loaded_plugin_names = set( sorted(fxt_tf_failure.extractors) + sorted(fxt_tf_failure.importers) From 29093ecd3e4233e3c2e3b2c66c08a55ed4f5f1ed Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Wed, 31 May 2023 16:49:21 +0900 Subject: [PATCH 09/10] Fix import error mocking Signed-off-by: Kim, Vinnam --- tests/unit/test_environment.py | 35 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py index 7a688f88a3..db6024603a 100644 --- a/tests/unit/test_environment.py +++ b/tests/unit/test_environment.py @@ -2,12 +2,15 @@ # # SPDX-License-Identifier: MIT +import builtins import sys import pytest from datumaro.components.environment import Environment +real_import = builtins.__import__ + class EnvironmentTest: @pytest.fixture @@ -34,26 +37,34 @@ def test_equivalance(self, fxt_lazy_import: Environment, fxt_no_lazy_import: Env assert sorted(fxt_lazy_import.validators) == sorted(fxt_no_lazy_import.validators) @pytest.fixture - def fxt_tf_failure(self, monkeypatch): - # Simulate `openvino.tools` and `tensorflow` import failure - monkeypatch.setitem(sys.modules, "openvino.tools", None) - monkeypatch.setitem(sys.modules, "tensorflow", None) + def fxt_tf_import_error(self, monkeypatch): + # Simulate `tensorflow` import failure + + def monkey_import_notfound(name, globals=None, locals=None, fromlist=(), level=0): + if name in ("tensorflow",): + raise ModuleNotFoundError(f"Mocked module not found {name}") + return real_import(name, globals=globals, locals=locals, fromlist=fromlist, level=level) + monkeypatch.delitem(sys.modules, "tensorflow", raising=False) + monkeypatch.setattr(builtins, "__import__", monkey_import_notfound) + + @pytest.fixture + def fxt_tf_failure_env(self, fxt_tf_import_error): with Environment.release_builtin_plugins(): env = Environment(use_lazy_import=True) _ = env.importers yield env - def test_extra_deps_req(self, fxt_tf_failure: Environment): + def test_extra_deps_req(self, fxt_tf_failure_env: Environment): """Plugins affected by the import failure: `ac` and `tf_detection_api`.""" loaded_plugin_names = set( - sorted(fxt_tf_failure.extractors) - + sorted(fxt_tf_failure.importers) - + sorted(fxt_tf_failure.launchers) - + sorted(fxt_tf_failure.exporters) - + sorted(fxt_tf_failure.generators) - + sorted(fxt_tf_failure.transforms) - + sorted(fxt_tf_failure.validators) + sorted(fxt_tf_failure_env.extractors) + + sorted(fxt_tf_failure_env.importers) + + sorted(fxt_tf_failure_env.launchers) + + sorted(fxt_tf_failure_env.exporters) + + sorted(fxt_tf_failure_env.generators) + + sorted(fxt_tf_failure_env.transforms) + + sorted(fxt_tf_failure_env.validators) ) assert "ac" not in loaded_plugin_names From 3c717963116f59a58b52b93d0e67a9e59b214f1d Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Wed, 31 May 2023 17:25:37 +0900 Subject: [PATCH 10/10] Fix mocking import error again Signed-off-by: Kim, Vinnam --- datumaro/components/environment.py | 7 +-- datumaro/plugins/specs.py | 21 +++++---- tests/unit/test_environment.py | 69 +++++++++++++++--------------- 3 files changed, 48 insertions(+), 49 deletions(-) diff --git a/datumaro/components/environment.py b/datumaro/components/environment.py index 46e9bb710e..60b6dc27dd 100644 --- a/datumaro/components/environment.py +++ b/datumaro/components/environment.py @@ -6,7 +6,6 @@ import importlib import logging as log import os.path as osp -from contextlib import contextmanager from functools import partial from inspect import isclass from typing import ( @@ -230,9 +229,9 @@ def _load_plugins(cls, module_names, *, importer, types=None): def _load_builtin_plugins(cls): """Load builtin plugins which will be imported lazily using plugin spec files""" if cls._builtin_plugins is None: - from datumaro.plugins.specs import LAZY_PLUGINS + from datumaro.plugins.specs import get_lazy_plugins - cls._builtin_plugins = LAZY_PLUGINS + cls._builtin_plugins = get_lazy_plugins() return cls._builtin_plugins @classmethod @@ -355,10 +354,8 @@ def _register(registry: PluginRegistry): return merged @classmethod - @contextmanager def release_builtin_plugins(cls): cls._builtin_plugins = None - yield DEFAULT_ENVIRONMENT = Environment() diff --git a/datumaro/plugins/specs.py b/datumaro/plugins/specs.py index 727047460d..01ad21db90 100644 --- a/datumaro/plugins/specs.py +++ b/datumaro/plugins/specs.py @@ -11,16 +11,19 @@ _SOURCE_DIR = _SOURCE_PATH.parent _SPECS_JSON_PATH = _SOURCE_DIR / "specs.json" -LAZY_PLUGINS = [ - plugin - for plugin in [ - get_lazy_plugin( - spec["import_path"], spec["plugin_name"], spec["plugin_type"], spec["extra_deps"] - ) - for spec in parse_json_file(str(_SPECS_JSON_PATH)) + +def get_lazy_plugins(): + return [ + plugin + for plugin in [ + get_lazy_plugin( + spec["import_path"], spec["plugin_name"], spec["plugin_type"], spec["extra_deps"] + ) + for spec in parse_json_file(str(_SPECS_JSON_PATH)) + ] + if plugin is not None ] - if plugin is not None -] + if __name__ == "__main__": from datumaro.components.environment import Environment diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py index db6024603a..f608c8f2cb 100644 --- a/tests/unit/test_environment.py +++ b/tests/unit/test_environment.py @@ -2,30 +2,30 @@ # # SPDX-License-Identifier: MIT -import builtins -import sys - import pytest +import datumaro.components.lazy_plugin from datumaro.components.environment import Environment -real_import = builtins.__import__ +real_import_module = datumaro.components.lazy_plugin.import_module class EnvironmentTest: @pytest.fixture def fxt_lazy_import(self): - with Environment.release_builtin_plugins(): - env = Environment(use_lazy_import=True) - _ = env.importers - yield env + Environment.release_builtin_plugins() + env = Environment(use_lazy_import=True) + _ = env.importers + yield env + Environment.release_builtin_plugins() @pytest.fixture def fxt_no_lazy_import(self): - with Environment.release_builtin_plugins(): - env = Environment(use_lazy_import=False) - _ = env.importers - yield env + Environment.release_builtin_plugins() + env = Environment(use_lazy_import=False) + _ = env.importers + yield env + Environment.release_builtin_plugins() def test_equivalance(self, fxt_lazy_import: Environment, fxt_no_lazy_import: Environment): assert sorted(fxt_lazy_import.extractors) == sorted(fxt_no_lazy_import.extractors) @@ -37,34 +37,33 @@ def test_equivalance(self, fxt_lazy_import: Environment, fxt_no_lazy_import: Env assert sorted(fxt_lazy_import.validators) == sorted(fxt_no_lazy_import.validators) @pytest.fixture - def fxt_tf_import_error(self, monkeypatch): - # Simulate `tensorflow` import failure - - def monkey_import_notfound(name, globals=None, locals=None, fromlist=(), level=0): - if name in ("tensorflow",): - raise ModuleNotFoundError(f"Mocked module not found {name}") - return real_import(name, globals=globals, locals=locals, fromlist=fromlist, level=level) + def fxt_tf_failure_env(self, monkeypatch): + def _patch(name, package=None): + if name == "tensorflow": + raise ImportError() + return real_import_module(name, package) - monkeypatch.delitem(sys.modules, "tensorflow", raising=False) - monkeypatch.setattr(builtins, "__import__", monkey_import_notfound) + monkeypatch.setattr(datumaro.components.lazy_plugin, "import_module", _patch) - @pytest.fixture - def fxt_tf_failure_env(self, fxt_tf_import_error): - with Environment.release_builtin_plugins(): - env = Environment(use_lazy_import=True) - _ = env.importers - yield env + Environment.release_builtin_plugins() + env = Environment(use_lazy_import=True) + _ = env.importers + yield env + Environment.release_builtin_plugins() - def test_extra_deps_req(self, fxt_tf_failure_env: Environment): + def test_extra_deps_req(self, fxt_tf_failure_env): """Plugins affected by the import failure: `ac` and `tf_detection_api`.""" + + env = fxt_tf_failure_env + loaded_plugin_names = set( - sorted(fxt_tf_failure_env.extractors) - + sorted(fxt_tf_failure_env.importers) - + sorted(fxt_tf_failure_env.launchers) - + sorted(fxt_tf_failure_env.exporters) - + sorted(fxt_tf_failure_env.generators) - + sorted(fxt_tf_failure_env.transforms) - + sorted(fxt_tf_failure_env.validators) + sorted(env.extractors) + + sorted(env.importers) + + sorted(env.launchers) + + sorted(env.exporters) + + sorted(env.generators) + + sorted(env.transforms) + + sorted(env.validators) ) assert "ac" not in loaded_plugin_names