diff --git a/.github/workflows/pre_merge.yaml b/.github/workflows/pre_merge.yaml index ceae7f0fc2..97a5dc3b16 100644 --- a/.github/workflows/pre_merge.yaml +++ b/.github/workflows/pre_merge.yaml @@ -94,6 +94,7 @@ jobs: - task: "instance_segmentation" - task: "semantic_segmentation" - task: "visual_prompting" + - task: "anomaly" name: Integration-Test-${{ matrix.task }}-py310 # This is what will cancel the job concurrency concurrency: diff --git a/pyproject.toml b/pyproject.toml index 09f521cbc5..29ec1343a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,18 @@ mmlab = [ "oss2==2.17.0", ] anomaly = [ - "anomalib==1.0.0", + # [FIXME] @ashwinvaidya17: Install using a temporary hot-fix commit due to a torchmetrics version conflict. + "anomalib @ git+https://github.com/openvinotoolkit/anomalib.git@e78091883a620229c277a79674a904d9f785f8d5", + # This is a dependency to avoid conflicts with installing the anomalib[core] option. + "av>=10.0.0", + "einops>=0.3.2", + "freia>=0.2", + "imgaug==0.4.0", + "kornia>=0.6.6,<0.6.10", + "matplotlib>=3.4.3", + "opencv-python>=4.5.3.56", + "pandas>=1.1.0", + "open-clip-torch>=2.23.0", ] [project.scripts] diff --git a/src/otx/algo/anomaly/padim.py b/src/otx/algo/anomaly/padim.py index de9bbb290b..4d70e02542 100644 --- a/src/otx/algo/anomaly/padim.py +++ b/src/otx/algo/anomaly/padim.py @@ -15,26 +15,26 @@ class Padim(OTXAnomaly, OTXModel, AnomalibPadim): """OTX Padim model. Args: - input_size (tuple[int, int], optional): Input size. Defaults to (256, 256). backbone (str, optional): Feature extractor backbone. Defaults to "resnet18". layers (list[str], optional): Feature extractor layers. Defaults to ["layer1", "layer2", "layer3"]. pre_trained (bool, optional): Pretrained backbone. Defaults to True. n_features (int | None, optional): Number of features. Defaults to None. + num_classes (int, optional): Anoamly don't use num_classes , + but OTXModel always receives num_classes, so need this. """ def __init__( self, - input_size: tuple[int, int] = (256, 256), backbone: str = "resnet18", layers: list[str] = ["layer1", "layer2", "layer3"], # noqa: B006 pre_trained: bool = True, n_features: int | None = None, + num_classes: int = 2, ) -> None: OTXAnomaly.__init__(self) - OTXModel.__init__(self, num_classes=2) + OTXModel.__init__(self, num_classes=num_classes) AnomalibPadim.__init__( self, - input_size=input_size, backbone=backbone, layers=layers, pre_trained=pre_trained, diff --git a/src/otx/algo/anomaly/stfpm.py b/src/otx/algo/anomaly/stfpm.py index 889ea33ab8..f77b70c4dd 100644 --- a/src/otx/algo/anomaly/stfpm.py +++ b/src/otx/algo/anomaly/stfpm.py @@ -21,21 +21,21 @@ class Stfpm(OTXAnomaly, OTXModel, AnomalibStfpm): Args: layers (Sequence[str]): Feature extractor layers. - input_size (tuple[int, int]): Input size. backbone (str, optional): Feature extractor backbone. Defaults to "resnet18". + num_classes (int, optional): Anoamly don't use num_classes , + but OTXModel always receives num_classes, so need this. """ def __init__( self, layers: Sequence[str] = ["layer1", "layer2", "layer3"], - input_size: tuple[int, int] = (256, 256), backbone: str = "resnet18", + num_classes: int = 2, ) -> None: OTXAnomaly.__init__(self) - OTXModel.__init__(self, num_classes=2) + OTXModel.__init__(self, num_classes=num_classes) AnomalibStfpm.__init__( self, - input_size=input_size, backbone=backbone, layers=layers, ) diff --git a/src/otx/cli/cli.py b/src/otx/cli/cli.py index 69d386e2bd..8aa0591936 100644 --- a/src/otx/cli/cli.py +++ b/src/otx/cli/cli.py @@ -413,7 +413,7 @@ def instantiate_model(self, model_config: Namespace) -> tuple: # Update num_classes if not self.get_config_value(self.config_init, "disable_infer_num_classes", False): num_classes = self.datamodule.label_info.num_classes - if num_classes != model_config.init_args.num_classes: + if hasattr(model_config.init_args, "num_classes") and num_classes != model_config.init_args.num_classes: warning_msg = ( f"The `num_classes` in dataset is {num_classes} " f"but, the `num_classes` of model is {model_config.init_args.num_classes}. " diff --git a/src/otx/core/data/entity/anomaly/classification.py b/src/otx/core/data/entity/anomaly/classification.py index 8cce41e722..cacdeb792c 100644 --- a/src/otx/core/data/entity/anomaly/classification.py +++ b/src/otx/core/data/entity/anomaly/classification.py @@ -53,7 +53,7 @@ def collate_fn( ) -> AnomalyClassificationDataBatch: """Collection function to collect `OTXDataEntity` into `OTXBatchDataEntity` in data loader.""" batch = super().collate_fn(entities) - images = tv_tensors.Image(data=torch.stack(batch.images, dim=0)) if stack_images else batch.images + images = tv_tensors.Image(data=torch.stack(tuple(batch.images), dim=0)) if stack_images else batch.images return AnomalyClassificationDataBatch( batch_size=batch.batch_size, images=images, diff --git a/src/otx/core/data/entity/anomaly/detection.py b/src/otx/core/data/entity/anomaly/detection.py index ad15d936d0..08a5e63805 100644 --- a/src/otx/core/data/entity/anomaly/detection.py +++ b/src/otx/core/data/entity/anomaly/detection.py @@ -56,7 +56,7 @@ def collate_fn( ) -> AnomalyDetectionDataBatch: """Collection function to collect `OTXDataEntity` into `OTXBatchDataEntity` in data loader.""" batch = super().collate_fn(entities) - images = tv_tensors.Image(data=torch.stack(batch.images, dim=0)) if stack_images else batch.images + images = tv_tensors.Image(data=torch.stack(tuple(batch.images), dim=0)) if stack_images else batch.images return AnomalyDetectionDataBatch( batch_size=batch.batch_size, images=images, diff --git a/src/otx/core/data/entity/anomaly/segmentation.py b/src/otx/core/data/entity/anomaly/segmentation.py index 1266c7529c..0ec64845cd 100644 --- a/src/otx/core/data/entity/anomaly/segmentation.py +++ b/src/otx/core/data/entity/anomaly/segmentation.py @@ -54,7 +54,7 @@ def collate_fn( ) -> AnomalySegmentationDataBatch: """Collection function to collect `OTXDataEntity` into `OTXBatchDataEntity` in data loader.""" batch = super().collate_fn(entities) - images = tv_tensors.Image(data=torch.stack(batch.images, dim=0)) if stack_images else batch.images + images = tv_tensors.Image(data=torch.stack(tuple(batch.images), dim=0)) if stack_images else batch.images return AnomalySegmentationDataBatch( batch_size=batch.batch_size, images=images, diff --git a/src/otx/core/model/module/anomaly/anomaly_lightning.py b/src/otx/core/model/module/anomaly/anomaly_lightning.py index f831cfd647..01fca867d9 100644 --- a/src/otx/core/model/module/anomaly/anomaly_lightning.py +++ b/src/otx/core/model/module/anomaly/anomaly_lightning.py @@ -134,7 +134,7 @@ class OTXAnomaly: def __init__(self) -> None: self.optimizer: list[OptimizerCallable] | OptimizerCallable = None self.scheduler: list[LRSchedulerCallable] | LRSchedulerCallable = None - self.input_size: list[int] = [256, 256] + self._input_size: tuple[int, int] = (256, 256) self.mean_values: tuple[float, float, float] = (0.0, 0.0, 0.0) self.scale_values: tuple[float, float, float] = (1.0, 1.0, 1.0) self.trainer: Trainer @@ -147,6 +147,19 @@ def __init__(self) -> None: self.image_metrics: AnomalibMetricCollection self.pixel_metrics: AnomalibMetricCollection + @property + def input_size(self) -> tuple[int, int]: + """Returns the input size of the model. + + Returns: + tuple[int, int]: The input size of the model as a tuple of (height, width). + """ + return self._input_size + + @input_size.setter + def input_size(self, value: tuple[int, int]) -> None: + self._input_size = value + @property def task(self) -> AnomalibTaskType: """Return the task type of the model.""" @@ -342,13 +355,13 @@ def state_dict(self) -> dict[str, Any]: """ state_dict = super().state_dict() # type: ignore[misc] # This is defined in OTXModel - state_dict["meta_info"] = self.meta_info # type: ignore[attr-defined] + state_dict["label_info"] = self.label_info # type: ignore[attr-defined] return state_dict def load_state_dict(self, ckpt: OrderedDict[str, Any], *args, **kwargs) -> None: """Pass the checkpoint to the anomaly model.""" ckpt = ckpt.get("state_dict", ckpt) - ckpt.pop("meta_info", None) # [TODO](ashwinvaidya17): Revisit this method when OTXModel is the lightning model + ckpt.pop("label_info", None) # [TODO](ashwinvaidya17): Revisit this method when OTXModel is the lightning model return super().load_state_dict(ckpt, *args, **kwargs) # type: ignore[misc] def forward( @@ -441,8 +454,9 @@ def export( """ min_val = self.normalization_metrics.state_dict()["min"].cpu().numpy().tolist() max_val = self.normalization_metrics.state_dict()["max"].cpu().numpy().tolist() + image_shape = (256, 256) if self.input_size is None else self.input_size exporter = _AnomalyModelExporter( - image_shape=(self.input_size[0], self.input_size[1]), + image_shape=image_shape, image_threshold=self.image_threshold.value.cpu().numpy().tolist(), pixel_threshold=self.pixel_threshold.value.cpu().numpy().tolist(), task=self.task, diff --git a/src/otx/engine/utils/auto_configurator.py b/src/otx/engine/utils/auto_configurator.py index 73ae76dcb6..90ce62ae6f 100644 --- a/src/otx/engine/utils/auto_configurator.py +++ b/src/otx/engine/utils/auto_configurator.py @@ -42,9 +42,9 @@ OTXTaskType.INSTANCE_SEGMENTATION: RECIPE_PATH / "instance_segmentation" / "maskrcnn_r50.yaml", OTXTaskType.ACTION_CLASSIFICATION: RECIPE_PATH / "action" / "action_classification" / "x3d.yaml", OTXTaskType.ACTION_DETECTION: RECIPE_PATH / "action" / "action_detection" / "x3d_fastrcnn.yaml", - OTXTaskType.ANOMALY_CLASSIFICATION: RECIPE_PATH / "anomaly" / "anomaly_classification" / "padim.yaml", - OTXTaskType.ANOMALY_SEGMENTATION: RECIPE_PATH / "anomaly" / "anomaly_segmentation" / "padim.yaml", - OTXTaskType.ANOMALY_DETECTION: RECIPE_PATH / "anomaly" / "anomaly_detection" / "padim.yaml", + OTXTaskType.ANOMALY_CLASSIFICATION: RECIPE_PATH / "anomaly_classification" / "padim.yaml", + OTXTaskType.ANOMALY_SEGMENTATION: RECIPE_PATH / "anomaly_segmentation" / "padim.yaml", + OTXTaskType.ANOMALY_DETECTION: RECIPE_PATH / "anomaly_detection" / "padim.yaml", OTXTaskType.VISUAL_PROMPTING: RECIPE_PATH / "visual_prompting" / "sam_tiny_vit.yaml", OTXTaskType.ZERO_SHOT_VISUAL_PROMPTING: RECIPE_PATH / "zero_shot_visual_prompting" / "sam_tiny_vit.yaml", } @@ -67,6 +67,7 @@ "common_semantic_segmentation_with_subset_dirs": [OTXTaskType.SEMANTIC_SEGMENTATION], "kinetics": [OTXTaskType.ACTION_CLASSIFICATION], "ava": [OTXTaskType.ACTION_DETECTION], + "mvtec": [OTXTaskType.ANOMALY_CLASSIFICATION, OTXTaskType.ANOMALY_DETECTION, OTXTaskType.ANOMALY_SEGMENTATION], } OVMODEL_PER_TASK = { diff --git a/src/otx/recipe/anomaly_classification/padim.yaml b/src/otx/recipe/anomaly_classification/padim.yaml index a759123a8e..75e037f551 100644 --- a/src/otx/recipe/anomaly_classification/padim.yaml +++ b/src/otx/recipe/anomaly_classification/padim.yaml @@ -1,9 +1,6 @@ model: class_path: otx.algo.anomaly.padim.Padim init_args: - input_size: - - 256 - - 256 layers: ["layer1", "layer2", "layer3"] backbone: "resnet18" pre_trained: True @@ -15,10 +12,15 @@ engine: callback_monitor: step # this has no effect as Padim does not need to be trained -data: ../../_base_/data/torchvision_base.yaml +data: ../_base_/data/torchvision_base.yaml overrides: + precision: 32 max_epochs: 1 limit_val_batches: 0 # this is set to 0 as the default dataloader does not have validation set. But this also means that the model will not give correct performance numbers + callbacks: + - class_path: otx.algo.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 data: task: ANOMALY_CLASSIFICATION config: diff --git a/src/otx/recipe/anomaly_classification/stfpm.yaml b/src/otx/recipe/anomaly_classification/stfpm.yaml index 9905505dd9..f518883423 100644 --- a/src/otx/recipe/anomaly_classification/stfpm.yaml +++ b/src/otx/recipe/anomaly_classification/stfpm.yaml @@ -1,9 +1,6 @@ model: class_path: otx.algo.anomaly.stfpm.Stfpm init_args: - input_size: - - 256 - - 256 layers: ["layer1", "layer2", "layer3"] backbone: "resnet18" @@ -21,7 +18,7 @@ engine: callback_monitor: train_loss_epoch # val loss is not available as there is no validation set from default dataloader -data: ../../_base_/data/torchvision_base.yaml +data: ../_base_/data/torchvision_base.yaml overrides: max_epochs: 100 limit_val_batches: 0 # this is set to 0 as the default dataloader does not have validation set. But this also means that the model will not give correct performance numbers @@ -29,6 +26,9 @@ overrides: - class_path: lightning.pytorch.callbacks.EarlyStopping init_args: patience: 5 + - class_path: otx.algo.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 data: task: ANOMALY_CLASSIFICATION config: diff --git a/src/otx/recipe/anomaly_detection/padim.yaml b/src/otx/recipe/anomaly_detection/padim.yaml index 0b8a73fbe5..fc62ee39d1 100644 --- a/src/otx/recipe/anomaly_detection/padim.yaml +++ b/src/otx/recipe/anomaly_detection/padim.yaml @@ -1,9 +1,6 @@ model: class_path: otx.algo.anomaly.padim.Padim init_args: - input_size: - - 256 - - 256 layers: ["layer1", "layer2", "layer3"] backbone: "resnet18" pre_trained: True @@ -15,10 +12,15 @@ engine: callback_monitor: step # this has no effect as Padim does not need to be trained -data: ../../_base_/data/torchvision_base.yaml +data: ../_base_/data/torchvision_base.yaml overrides: + precision: 32 max_epochs: 1 limit_val_batches: 0 # this is set to 0 as the default dataloader does not have validation set. But this also means that the model will not give correct performance numbers + callbacks: + - class_path: otx.algo.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 data: task: ANOMALY_DETECTION config: diff --git a/src/otx/recipe/anomaly_detection/stfpm.yaml b/src/otx/recipe/anomaly_detection/stfpm.yaml index 7cefa80329..221865aab9 100644 --- a/src/otx/recipe/anomaly_detection/stfpm.yaml +++ b/src/otx/recipe/anomaly_detection/stfpm.yaml @@ -1,9 +1,6 @@ model: class_path: otx.algo.anomaly.stfpm.Stfpm init_args: - input_size: - - 256 - - 256 layers: ["layer1", "layer2", "layer3"] backbone: "resnet18" @@ -21,7 +18,7 @@ engine: callback_monitor: train_loss_epoch # val loss is not available as there is no validation set from default dataloader -data: ../../_base_/data/torchvision_base.yaml +data: ../_base_/data/torchvision_base.yaml overrides: max_epochs: 100 limit_val_batches: 0 # this is set to 0 as the default dataloader does not have validation set. But this also means that the model will not give correct performance numbers @@ -29,6 +26,9 @@ overrides: - class_path: lightning.pytorch.callbacks.EarlyStopping init_args: patience: 5 + - class_path: otx.algo.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 data: task: ANOMALY_DETECTION config: diff --git a/src/otx/recipe/anomaly_segmentation/padim.yaml b/src/otx/recipe/anomaly_segmentation/padim.yaml index a2035f8001..3ed2934eb8 100644 --- a/src/otx/recipe/anomaly_segmentation/padim.yaml +++ b/src/otx/recipe/anomaly_segmentation/padim.yaml @@ -1,9 +1,6 @@ model: class_path: otx.algo.anomaly.padim.Padim init_args: - input_size: - - 256 - - 256 layers: ["layer1", "layer2", "layer3"] backbone: "resnet18" pre_trained: True @@ -15,10 +12,15 @@ engine: callback_monitor: step # this has no effect as Padim does not need to be trained -data: ../../_base_/data/torchvision_base.yaml +data: ../_base_/data/torchvision_base.yaml overrides: + precision: 32 max_epochs: 1 limit_val_batches: 0 # this is set to 0 as the default dataloader does not have validation set. But this also means that the model will not give correct performance numbers + callbacks: + - class_path: otx.algo.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 data: task: ANOMALY_SEGMENTATION config: diff --git a/src/otx/recipe/anomaly_segmentation/stfpm.yaml b/src/otx/recipe/anomaly_segmentation/stfpm.yaml index 9cb153e24a..a045fdd348 100644 --- a/src/otx/recipe/anomaly_segmentation/stfpm.yaml +++ b/src/otx/recipe/anomaly_segmentation/stfpm.yaml @@ -1,9 +1,6 @@ model: class_path: otx.algo.anomaly.stfpm.Stfpm init_args: - input_size: - - 256 - - 256 layers: ["layer1", "layer2", "layer3"] backbone: "resnet18" @@ -21,7 +18,7 @@ engine: callback_monitor: train_loss_epoch # val loss is not available as there is no validation set from default dataloader -data: ../../_base_/data/torchvision_base.yaml +data: ../_base_/data/torchvision_base.yaml overrides: max_epochs: 100 limit_val_batches: 0 # this is set to 0 as the default dataloader does not have validation set. But this also means that the model will not give correct performance numbers @@ -29,6 +26,9 @@ overrides: - class_path: lightning.pytorch.callbacks.EarlyStopping init_args: patience: 5 + - class_path: otx.algo.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 data: task: ANOMALY_SEGMENTATION config: diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/00.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/00.png new file mode 100644 index 0000000000..0650051dc1 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/00.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/01.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/01.png new file mode 100644 index 0000000000..3193a7fcab Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/01.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/02.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/02.png new file mode 100644 index 0000000000..7edbb42a0f Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/02.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/03.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/03.png new file mode 100644 index 0000000000..78d290ed8d Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/03.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/04.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/04.png new file mode 100644 index 0000000000..ae404f0494 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/04.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/05.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/05.png new file mode 100644 index 0000000000..c60c760b1e Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/05.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/06.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/06.png new file mode 100644 index 0000000000..dffa48d8ef Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/06.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/07.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/07.png new file mode 100644 index 0000000000..2412619f14 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/07.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/08.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/08.png new file mode 100644 index 0000000000..361ff79b7c Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/08.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/09.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/09.png new file mode 100644 index 0000000000..ef51fd2c26 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/09.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/10.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/10.png new file mode 100644 index 0000000000..3c9c9a4122 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/10.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/11.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/11.png new file mode 100644 index 0000000000..9e154d9091 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/11.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/12.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/12.png new file mode 100644 index 0000000000..44726369cd Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/12.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/13.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/13.png new file mode 100644 index 0000000000..a665aeb5ff Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/13.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/14.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/14.png new file mode 100644 index 0000000000..f5311fd3ab Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/14.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/15.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/15.png new file mode 100644 index 0000000000..379fc95db9 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/15.png differ diff --git a/tests/assets/anomaly_hazelnut/ground_truth/colour/16.png b/tests/assets/anomaly_hazelnut/ground_truth/colour/16.png new file mode 100644 index 0000000000..4f5bd50acc Binary files /dev/null and b/tests/assets/anomaly_hazelnut/ground_truth/colour/16.png differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/00.jpg b/tests/assets/anomaly_hazelnut/test/colour/00.jpg new file mode 100644 index 0000000000..ceb643434a Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/00.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/01.jpg b/tests/assets/anomaly_hazelnut/test/colour/01.jpg new file mode 100644 index 0000000000..a9ab8c84bd Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/01.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/02.jpg b/tests/assets/anomaly_hazelnut/test/colour/02.jpg new file mode 100644 index 0000000000..7ffa06dab0 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/02.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/03.jpg b/tests/assets/anomaly_hazelnut/test/colour/03.jpg new file mode 100644 index 0000000000..5768694ccc Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/03.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/04.jpg b/tests/assets/anomaly_hazelnut/test/colour/04.jpg new file mode 100644 index 0000000000..39a9c897fd Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/04.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/05.jpg b/tests/assets/anomaly_hazelnut/test/colour/05.jpg new file mode 100644 index 0000000000..04bf65e4ef Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/05.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/06.jpg b/tests/assets/anomaly_hazelnut/test/colour/06.jpg new file mode 100644 index 0000000000..cd6e1a3192 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/06.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/07.jpg b/tests/assets/anomaly_hazelnut/test/colour/07.jpg new file mode 100644 index 0000000000..e3a9be8b5d Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/07.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/08.jpg b/tests/assets/anomaly_hazelnut/test/colour/08.jpg new file mode 100644 index 0000000000..6e7e451ab1 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/08.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/09.jpg b/tests/assets/anomaly_hazelnut/test/colour/09.jpg new file mode 100644 index 0000000000..339a8a0dc0 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/09.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/10.jpg b/tests/assets/anomaly_hazelnut/test/colour/10.jpg new file mode 100644 index 0000000000..c3009b8ffd Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/10.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/11.jpg b/tests/assets/anomaly_hazelnut/test/colour/11.jpg new file mode 100644 index 0000000000..fbdf84c370 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/11.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/12.jpg b/tests/assets/anomaly_hazelnut/test/colour/12.jpg new file mode 100644 index 0000000000..bda39303e0 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/12.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/13.jpg b/tests/assets/anomaly_hazelnut/test/colour/13.jpg new file mode 100644 index 0000000000..28a5b9c53c Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/13.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/14.jpg b/tests/assets/anomaly_hazelnut/test/colour/14.jpg new file mode 100644 index 0000000000..78bb3b88ae Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/14.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/15.jpg b/tests/assets/anomaly_hazelnut/test/colour/15.jpg new file mode 100644 index 0000000000..c654528134 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/15.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/colour/16.jpg b/tests/assets/anomaly_hazelnut/test/colour/16.jpg new file mode 100644 index 0000000000..e6bd3cd8f5 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/colour/16.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/good/04.jpg b/tests/assets/anomaly_hazelnut/test/good/04.jpg new file mode 100644 index 0000000000..6fc30ac701 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/good/04.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/good/05.jpg b/tests/assets/anomaly_hazelnut/test/good/05.jpg new file mode 100644 index 0000000000..1147ab6fe5 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/good/05.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/good/13.jpg b/tests/assets/anomaly_hazelnut/test/good/13.jpg new file mode 100644 index 0000000000..7a83cfbd97 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/good/13.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/good/23.jpg b/tests/assets/anomaly_hazelnut/test/good/23.jpg new file mode 100644 index 0000000000..667ed45588 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/good/23.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/good/25.jpg b/tests/assets/anomaly_hazelnut/test/good/25.jpg new file mode 100644 index 0000000000..1563721035 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/good/25.jpg differ diff --git a/tests/assets/anomaly_hazelnut/test/good/28.jpg b/tests/assets/anomaly_hazelnut/test/good/28.jpg new file mode 100644 index 0000000000..c17e1ba90a Binary files /dev/null and b/tests/assets/anomaly_hazelnut/test/good/28.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/00.jpg b/tests/assets/anomaly_hazelnut/train/good/00.jpg new file mode 100644 index 0000000000..092ce68bea Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/00.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/01.jpg b/tests/assets/anomaly_hazelnut/train/good/01.jpg new file mode 100644 index 0000000000..7b1e889780 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/01.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/02.jpg b/tests/assets/anomaly_hazelnut/train/good/02.jpg new file mode 100644 index 0000000000..0c1864a0bb Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/02.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/03.jpg b/tests/assets/anomaly_hazelnut/train/good/03.jpg new file mode 100644 index 0000000000..fe5529ab90 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/03.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/06.jpg b/tests/assets/anomaly_hazelnut/train/good/06.jpg new file mode 100644 index 0000000000..4cc649d850 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/06.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/07.jpg b/tests/assets/anomaly_hazelnut/train/good/07.jpg new file mode 100644 index 0000000000..4e0b7576ad Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/07.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/08.jpg b/tests/assets/anomaly_hazelnut/train/good/08.jpg new file mode 100644 index 0000000000..a644e41cc7 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/08.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/09.jpg b/tests/assets/anomaly_hazelnut/train/good/09.jpg new file mode 100644 index 0000000000..216bca7a76 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/09.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/10.jpg b/tests/assets/anomaly_hazelnut/train/good/10.jpg new file mode 100644 index 0000000000..f6cac4df67 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/10.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/11.jpg b/tests/assets/anomaly_hazelnut/train/good/11.jpg new file mode 100644 index 0000000000..3068e623d8 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/11.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/12.jpg b/tests/assets/anomaly_hazelnut/train/good/12.jpg new file mode 100644 index 0000000000..47483ba6eb Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/12.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/14.jpg b/tests/assets/anomaly_hazelnut/train/good/14.jpg new file mode 100644 index 0000000000..c85eb5dd0d Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/14.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/15.jpg b/tests/assets/anomaly_hazelnut/train/good/15.jpg new file mode 100644 index 0000000000..6269cc87c0 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/15.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/16.jpg b/tests/assets/anomaly_hazelnut/train/good/16.jpg new file mode 100644 index 0000000000..47247c4849 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/16.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/17.jpg b/tests/assets/anomaly_hazelnut/train/good/17.jpg new file mode 100644 index 0000000000..8f87931993 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/17.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/18.jpg b/tests/assets/anomaly_hazelnut/train/good/18.jpg new file mode 100644 index 0000000000..935abb536f Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/18.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/19.jpg b/tests/assets/anomaly_hazelnut/train/good/19.jpg new file mode 100644 index 0000000000..25bc41f7b3 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/19.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/20.jpg b/tests/assets/anomaly_hazelnut/train/good/20.jpg new file mode 100644 index 0000000000..991e84229f Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/20.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/21.jpg b/tests/assets/anomaly_hazelnut/train/good/21.jpg new file mode 100644 index 0000000000..8324520b24 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/21.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/22.jpg b/tests/assets/anomaly_hazelnut/train/good/22.jpg new file mode 100644 index 0000000000..407c67c78b Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/22.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/24.jpg b/tests/assets/anomaly_hazelnut/train/good/24.jpg new file mode 100644 index 0000000000..201148307f Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/24.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/26.jpg b/tests/assets/anomaly_hazelnut/train/good/26.jpg new file mode 100644 index 0000000000..7ee8f39458 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/26.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/27.jpg b/tests/assets/anomaly_hazelnut/train/good/27.jpg new file mode 100644 index 0000000000..40163b45d4 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/27.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/29.jpg b/tests/assets/anomaly_hazelnut/train/good/29.jpg new file mode 100644 index 0000000000..f6516ecc9f Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/29.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/30.jpg b/tests/assets/anomaly_hazelnut/train/good/30.jpg new file mode 100644 index 0000000000..1f2fa4ccd0 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/30.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/31.jpg b/tests/assets/anomaly_hazelnut/train/good/31.jpg new file mode 100644 index 0000000000..0d3679de31 Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/31.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/32.jpg b/tests/assets/anomaly_hazelnut/train/good/32.jpg new file mode 100644 index 0000000000..ead1e4198e Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/32.jpg differ diff --git a/tests/assets/anomaly_hazelnut/train/good/33.jpg b/tests/assets/anomaly_hazelnut/train/good/33.jpg new file mode 100644 index 0000000000..e850eb144b Binary files /dev/null and b/tests/assets/anomaly_hazelnut/train/good/33.jpg differ diff --git a/tests/integration/api/test_auto_configuration.py b/tests/integration/api/test_auto_configuration.py index 1b45bdf423..217a1e3868 100644 --- a/tests/integration/api/test_auto_configuration.py +++ b/tests/integration/api/test_auto_configuration.py @@ -32,6 +32,8 @@ def test_auto_configuration( pytest.skip( reason="H-labels require num_multiclass_head, num_multilabel_classes, which skip until we have the ability to automate this.", ) + if task.lower().startswith("anomaly"): + pytest.skip(reason="This will be added in a future pipeline behavior.") tmp_path_train = tmp_path / f"auto_train_{task}" data_root = fxt_target_dataset_per_task[task.lower()] diff --git a/tests/integration/api/test_engine_api.py b/tests/integration/api/test_engine_api.py index 13d3b6888e..6ee5ea67c7 100644 --- a/tests/integration/api/test_engine_api.py +++ b/tests/integration/api/test_engine_api.py @@ -37,8 +37,6 @@ def test_engine_from_config( pytest.skip( reason="H-labels require num_multiclass_head, num_multilabel_classes, which skip until we have the ability to automate this.", ) - if "anomaly" in task.lower(): - pytest.skip(reason="There's no dataset for anomaly tasks.") tmp_path_train = tmp_path / task engine = Engine.from_config( @@ -70,6 +68,9 @@ def test_engine_from_config( OTXTaskType.ACTION_DETECTION, OTXTaskType.H_LABEL_CLS, OTXTaskType.ROTATED_DETECTION, + OTXTaskType.ANOMALY_CLASSIFICATION, + OTXTaskType.ANOMALY_DETECTION, + OTXTaskType.ANOMALY_SEGMENTATION, ]: return diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 295043fac0..0339c6378d 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -129,6 +129,9 @@ def test_otx_e2e( "dino_v2", "instance_segmentation", "action", + "anomaly_classification", + "anomaly_detection", + "anomaly_segmentation", ] ): return @@ -372,6 +375,9 @@ def test_otx_ov_test( "h_label_cls", "visual_prompting", "zero_shot_visual_prompting", + "anomaly_classification", + "anomaly_detection", + "anomaly_segmentation", ]: # OMZ doesn't have proper model for Pytorch MaskRCNN interface # TODO(Kirill): Need to change this test when export enabled #noqa: TD003 @@ -469,4 +475,8 @@ def test_otx_hpo_e2e( ) hpo_work_dor = latest_dir / "hpo" assert hpo_work_dor.exists() + # Anomaly doesn't do validation. Check just there is no error. + if task.startswith("anomaly"): + return + assert len([val for val in hpo_work_dor.rglob("*.json") if str(val.stem).isdigit()]) == 2 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3c9d706c30..22a75105e2 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -66,6 +66,8 @@ def get_task_list(task: str) -> list[OTXTaskType]: return [OTXTaskType.ACTION_CLASSIFICATION, OTXTaskType.ACTION_DETECTION] if task == "visual_prompting": return [OTXTaskType.VISUAL_PROMPTING, OTXTaskType.ZERO_SHOT_VISUAL_PROMPTING] + if task == "anomaly": + return [OTXTaskType.ANOMALY_CLASSIFICATION, OTXTaskType.ANOMALY_DETECTION, OTXTaskType.ANOMALY_SEGMENTATION] return [OTXTaskType(task.upper())] @@ -132,6 +134,9 @@ def fxt_target_dataset_per_task() -> dict: "action_detection": "tests/assets/action_detection_dataset/", "visual_prompting": "tests/assets/car_tree_bug", "zero_shot_visual_prompting": "tests/assets/car_tree_bug_zero_shot", + "anomaly_classification": "tests/assets/anomaly_hazelnut", + "anomaly_detection": "tests/assets/anomaly_hazelnut", + "anomaly_segmentation": "tests/assets/anomaly_hazelnut", } @@ -157,4 +162,7 @@ def fxt_cli_override_command_per_task() -> dict: ], "visual_prompting": [], "zero_shot_visual_prompting": [], + "anomaly_classification": ["--limit_val_batches", "0"], + "anomaly_detection": ["--limit_val_batches", "0"], + "anomaly_segmentation": ["--limit_val_batches", "0"], } diff --git a/tox.ini b/tox.ini index 1f73249b96..c1c7ab0e9f 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ task = instance_segmentation: "instance_segmentation" semantic_segmentation: "semantic_segmentation" visual_prompting: "visual_prompting" + anomaly: "anomaly" passenv = ftp_proxy HTTP_PROXY @@ -38,8 +39,6 @@ deps = commands_pre = ; [TODO]: Needs to be fixed so that this is not duplicated for each test run otx install -v - ; temporary as Anomalib v1 is not available on PyPI - pip install git+https://github.com/openvinotoolkit/anomalib.git@cbb623e33876e446b7788375cc355e3a3dd44cef commands = ; Run Unit-Test with coverage report. pytest tests/unit \ @@ -50,7 +49,7 @@ commands = {posargs} -[testenv:integration-test-{all, action, classification, detection, rotated_detection, instance_segmentation, semantic_segmentation, visual_prompting}] +[testenv:integration-test-{all, action, classification, detection, rotated_detection, instance_segmentation, semantic_segmentation, visual_prompting, anomaly}] setenv = CUBLAS_WORKSPACE_CONFIG=:4096:8 deps = @@ -58,8 +57,6 @@ deps = commands_pre = ; [TODO]: Needs to be fixed so that this is not duplicated for each test run otx install -v - ; temporary as Anomalib v1 is not available on PyPI - pip install git+https://github.com/openvinotoolkit/anomalib.git@cbb623e33876e446b7788375cc355e3a3dd44cef commands = python -m pytest tests/integration -ra --showlocals --csv={toxworkdir}/{envname}.csv --task {[testenv]task} --open-subprocess {posargs} @@ -69,8 +66,6 @@ deps = commands_pre = ; [TODO]: Needs to be fixed so that this is not duplicated for each test run otx install -v - ; temporary as Anomalib v1 is not available on PyPI - pip install git+https://github.com/openvinotoolkit/anomalib.git@cbb623e33876e446b7788375cc355e3a3dd44cef commands = pytest -ra --showlocals --csv={toxworkdir}/{envname}.csv {posargs:tests/perf}