From 3cd2a4ff9ae369453dfc5de38541da264430e1d6 Mon Sep 17 00:00:00 2001 From: Thomas Lips <37955681+tlpss@users.noreply.github.com> Date: Tue, 24 Jan 2023 23:47:40 +0100 Subject: [PATCH 01/45] replace miniconda by micromamba and cache environments (#21) * try micromamba * trigger ci again * trigger once more --- .github/workflows/pytest.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 251ba87..f932b68 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -10,25 +10,14 @@ jobs: runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v2 - - name: Cache conda - uses: actions/cache@v2 - env: - # Increase this value to reset cache if etc/example-environment.yml has not changed - CACHE_NUMBER: 1 + - name: install conda env with micromamba + uses: mamba-org/provision-with-micromamba@main with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yaml') }} - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: keypoint-detection # name of env channel-priority: strict - auto-activate-base: false environment-file: environment.yaml - use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + cache-env: true - name: Conda list shell: bash -l {0} - run: conda list - name: pytest shell: bash -l {0} From ceec1e4c75126bd1d365ba539d3335c1f5d25301 Mon Sep 17 00:00:00 2001 From: Thomas Lips <37955681+tlpss@users.noreply.github.com> Date: Sun, 19 Mar 2023 18:25:19 +0100 Subject: [PATCH 02/45] lock pytorch lightning to < 2.0 for now to avoid the breaking changes --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d58fcab..2d2b07d 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ install_requires=[ "torch>=0.10", "torchvision>=0.11", - "pytorch-lightning>=1.5.10", + "pytorch-lightning>=1.5.10,<=1.9.4", # PL 2.0 has breaking changes that need to be incorporated "torchmetrics>=0.7", "wandb>=0.13.7", # artifact bug https://github.com/wandb/wandb/issues/4500 "timm>=0.6.11", # requires smallsized convnext models From da41920f2791018f0413e9606f96b55956f15929 Mon Sep 17 00:00:00 2001 From: tlpss Date: Sun, 26 Mar 2023 22:02:30 +0200 Subject: [PATCH 03/45] bump pre-commit to fix poetry issue --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8959224..5a57043 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: files: \.py$ - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort name: isort - sort imports From 6ce25c3f327ea8d1a99594eab89b2e14f9221089 Mon Sep 17 00:00:00 2001 From: tlpss Date: Sun, 26 Mar 2023 22:04:23 +0200 Subject: [PATCH 04/45] format --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d2b07d..8cc848e 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ install_requires=[ "torch>=0.10", "torchvision>=0.11", - "pytorch-lightning>=1.5.10,<=1.9.4", # PL 2.0 has breaking changes that need to be incorporated + "pytorch-lightning>=1.5.10,<=1.9.4", # PL 2.0 has breaking changes that need to be incorporated "torchmetrics>=0.7", "wandb>=0.13.7", # artifact bug https://github.com/wandb/wandb/issues/4500 "timm>=0.6.11", # requires smallsized convnext models From b493e6107f7a35adb08b9a7df5ed1982d4f5c74b Mon Sep 17 00:00:00 2001 From: Lucas Van Dijck <78962099+lucasvandijck@users.noreply.github.com> Date: Sun, 26 Mar 2023 22:18:28 +0200 Subject: [PATCH 05/45] Add MobileNetV3 backbone (#24) * added image_size argument * added mnv3 backbone * Added another branch by accident, removing the changes * Re-added datamodule.py * re-used convnext UpSamplingBlock * format --------- Co-authored-by: tlpss --- .../models/backbones/backbone_factory.py | 3 +- .../models/backbones/mobilenetv3.py | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 keypoint_detection/models/backbones/mobilenetv3.py diff --git a/keypoint_detection/models/backbones/backbone_factory.py b/keypoint_detection/models/backbones/backbone_factory.py index e959df3..6ecf5e0 100644 --- a/keypoint_detection/models/backbones/backbone_factory.py +++ b/keypoint_detection/models/backbones/backbone_factory.py @@ -5,13 +5,14 @@ from keypoint_detection.models.backbones.convnext_unet import ConvNeXtUnet from keypoint_detection.models.backbones.dilated_cnn import DilatedCnn from keypoint_detection.models.backbones.maxvit_unet import MaxVitUnet +from keypoint_detection.models.backbones.mobilenetv3 import MobileNetV3 from keypoint_detection.models.backbones.s3k import S3K from keypoint_detection.models.backbones.unet import Unet class BackboneFactory: # TODO: how to auto-register with __init__subclass over multiple files? - registered_backbone_classes: List[Backbone] = [Unet, ConvNeXtUnet, MaxVitUnet, S3K, DilatedCnn] + registered_backbone_classes: List[Backbone] = [Unet, ConvNeXtUnet, MaxVitUnet, S3K, DilatedCnn, MobileNetV3] @staticmethod def create_backbone(backbone_type: str, **kwargs) -> Backbone: diff --git a/keypoint_detection/models/backbones/mobilenetv3.py b/keypoint_detection/models/backbones/mobilenetv3.py new file mode 100644 index 0000000..d498727 --- /dev/null +++ b/keypoint_detection/models/backbones/mobilenetv3.py @@ -0,0 +1,44 @@ +"""A MobileNetV3-based backbone. +""" +import timm +import torch +import torch.nn as nn + +from keypoint_detection.models.backbones.base_backbone import Backbone +from keypoint_detection.models.backbones.convnext_unet import UpSamplingBlock + + +class MobileNetV3(Backbone): + """ + Pretrained MobileNetV3 using the large_100 model with 3.4M parameters. + """ + + def __init__(self, **kwargs): + super().__init__() + self.encoder = timm.create_model("mobilenetv3_large_100", pretrained=True, features_only=True) + self.decoder_blocks = nn.ModuleList() + for i in range(1, len(self.encoder.feature_info.info)): + channels_in, skip_channels_in = ( + self.encoder.feature_info.info[-i]["num_chs"], + self.encoder.feature_info.info[-i - 1]["num_chs"], + ) + block = UpSamplingBlock(channels_in, skip_channels_in, skip_channels_in, 3) + self.decoder_blocks.append(block) + + self.final_upsampling_block = nn.Sequential( + nn.UpsamplingBilinear2d(scale_factor=2), + nn.Conv2d(skip_channels_in, skip_channels_in, 3, padding="same"), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + features = self.encoder(x) + + x = features.pop() + for block in self.decoder_blocks: + x = block(x, features.pop()) + x = self.final_upsampling_block(x) + + return x + + def get_n_channels_out(self): + return self.encoder.feature_info.info[0]["num_chs"] From 33bcb5fbbeaf5683a30479f4836fd3758dbc24ca Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 4 Jul 2023 09:27:31 +0200 Subject: [PATCH 06/45] use new channels for conda installation of torch --- environment.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/environment.yaml b/environment.yaml index e3fd1f1..91fc4ae 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,11 +1,12 @@ name: keypoint-detection # to update an existing environment: conda env update -n --file channels: - pytorch + - nvidia - conda-forge dependencies: - - cudatoolkit=11.3 - python=3.9 - - pytorch + - pytorch=1.13 + - pytorch-cuda=11.7 - torchvision - pip - pip: From 9fb540bc920c9947c1a4279c91303604aad312ca Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 4 Jul 2023 09:47:29 +0200 Subject: [PATCH 07/45] fix bug with augmentations if no validation dataset is provided --- keypoint_detection/data/datamodule.py | 16 +++++++++--- test/integration_test.sh | 4 +-- test/test_datamodule.py | 35 +++++++++++++++++---------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/keypoint_detection/data/datamodule.py b/keypoint_detection/data/datamodule.py index 27bc1c9..333ef48 100644 --- a/keypoint_detection/data/datamodule.py +++ b/keypoint_detection/data/datamodule.py @@ -5,7 +5,7 @@ import numpy as np import pytorch_lightning as pl import torch -from torch.utils.data import DataLoader +from torch.utils.data import DataLoader, Subset from keypoint_detection.data.augmentations import MultiChannelKeypointsCompose from keypoint_detection.data.coco_dataset import COCOKeypointsDataset @@ -61,6 +61,7 @@ def __init__( self.batch_size = batch_size self.num_workers = num_workers self.augment_train = augment_train + self.train_dataset = COCOKeypointsDataset(json_dataset_path, keypoint_channel_configuration, **kwargs) self.validation_dataset = None @@ -78,17 +79,24 @@ def __init__( if json_test_dataset_path: self.test_dataset = COCOKeypointsDataset(json_test_dataset_path, keypoint_channel_configuration, **kwargs) + # create the transforms if needed and set them to the datasets if augment_train: - img_size = self.train_dataset[0][0].shape[1] # assume rectangular! + img_width, img_height = self.train_dataset[0][0].shape[1], self.train_dataset[0][0].shape[2] train_transform = MultiChannelKeypointsCompose( [ A.ColorJitter(), A.RandomRotate90(), A.HorizontalFlip(), - A.RandomResizedCrop(img_size, img_size, scale=(0.8, 1.0), ratio=(0.95, 1.0)), + A.RandomResizedCrop(img_height, img_width, scale=(0.8, 1.0), ratio=(0.95, 1.0)), ] ) - self.train_dataset.transform = train_transform + if isinstance(self.train_dataset, COCOKeypointsDataset): + self.train_dataset.transform = train_transform + elif isinstance(self.train_dataset, Subset): + # if the train dataset is a subset, we need to set the transform to the underlying dataset + # otherwise the transform will not be applied.. + assert isinstance(self.train_dataset.dataset, COCOKeypointsDataset) + self.train_dataset.dataset.transform = train_transform @staticmethod def _split_dataset(dataset, validation_split_ratio): diff --git a/test/integration_test.sh b/test/integration_test.sh index 2d520f5..96b9380 100644 --- a/test/integration_test.sh +++ b/test/integration_test.sh @@ -5,5 +5,5 @@ # make sure to remove all trailing spaces from the command, as this would result in an error when using bash. python keypoint_detection/train/train.py \ --keypoint_channel_configuration "box_corner0= box_corner1 = box_corner2= box_corner3; flap_corner0 ; flap_corner2" \ ---json_dataset_path "test/test_dataset/coco_dataset.json" --batch_size 2 --wandb_project "keypoint-detector-integration-test" \ ---max_epochs 50 --early_stopping_relative_threshold -1.0 --log_every_n_steps 1 --accelerator="gpu" --devices 1 --precision 16 +--json_dataset_path "test/test_dataset/coco_dataset.json" --json_validation_dataset_path "test/test_dataset/coco_dataset.json" --batch_size 2 --wandb_project "keypoint-detector-integration-test" \ +--max_epochs 50 --early_stopping_relative_threshold -1.0 --log_every_n_steps 1 --accelerator="gpu" --devices 1 --precision 16 --augment_train diff --git a/test/test_datamodule.py b/test/test_datamodule.py index c692bf4..8cdcde6 100644 --- a/test/test_datamodule.py +++ b/test/test_datamodule.py @@ -4,6 +4,7 @@ from test.configuration import DEFAULT_HPARAMS, TEST_PARAMS import torch +import torch.utils.data from keypoint_detection.data.datamodule import KeypointsDataModule @@ -51,29 +52,37 @@ def test_batch_format(self): self.assertIsInstance(ch2[0], torch.Tensor) def test_augmentations_result_in_different_image(self): + # get the dataset through the datamodule + # cannot use dataloader directly bc it shuffles the dataset. random.seed(2022) torch.manual_seed(2022) hparams = copy.deepcopy(DEFAULT_HPARAMS) + hparams["augment_train"] = False + module = KeypointsDataModule(**hparams) - train_dataloader = module.train_dataloader() + no_aug_train_dataloader = module.train_dataloader() + no_aug_dataset = no_aug_train_dataloader.dataset - batch = next(iter(train_dataloader)) - img, _ = batch + img, _ = no_aug_dataset[0] - hparams = copy.deepcopy(DEFAULT_HPARAMS) + # reset seeds to obtain the same dataset order + # and get the dataset again but now with augmentations + random.seed(2022) + torch.manual_seed(2022) + hparams = copy.deepcopy(hparams) hparams["augment_train"] = True - module = KeypointsDataModule(**hparams) - train_dataloader = module.train_dataloader() + aug_module = KeypointsDataModule(**hparams) + aug_train_dataloader = aug_module.train_dataloader() + aug_dataset = aug_train_dataloader.dataset - dissimilar_batches = 0 - # iterate over a few batches - # bc none of the augmentations is applied with 100% probability + dissimilar_images = 0 + # iterate a few times over the dataset to check that the augmentations are applied + # bc none of the augmentations is applied with 100% probability so some batches could be equal # and finding a seed that triggers them could change if you change the augmentations for _ in range(5): - batch = next(iter(train_dataloader)) - transformed_img, _ = batch + transformed_img, _ = aug_dataset[0] # check both images are not equal. - dissimilar_batches += 1 * (torch.linalg.norm(img - transformed_img) != 0.0) + dissimilar_images += 1 * (torch.linalg.norm(img - transformed_img) != 0.0) - self.assertTrue(dissimilar_batches > 0) + self.assertTrue(dissimilar_images > 0) From 5463e6fee924cc4b46a4c65b7205bb010e0ea291 Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 4 Jul 2023 09:47:49 +0200 Subject: [PATCH 08/45] handle rgb-a images --- keypoint_detection/data/coco_dataset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/keypoint_detection/data/coco_dataset.py b/keypoint_detection/data/coco_dataset.py index 733990b..c006a97 100644 --- a/keypoint_detection/data/coco_dataset.py +++ b/keypoint_detection/data/coco_dataset.py @@ -56,7 +56,7 @@ def __init__( detect_non_visible_keypoints: bool = True, transform: A.Compose = None, imageloader: ImageLoader = None, - **kwargs + **kwargs, ): super().__init__(imageloader) @@ -88,9 +88,11 @@ def __getitem__(self, index) -> Tuple[torch.Tensor, IMG_KEYPOINTS_TYPE]: image_path = self.dataset_dir_path / self.dataset[index][0] image = self.image_loader.get_image(str(image_path), index) + # remove a-channel if needed + if image.shape[2] == 4: + image = image[..., :3] keypoints = self.dataset[index][1] - if self.transform: transformed = self.transform(image=image, keypoints=keypoints) image, keypoints = transformed["image"], transformed["keypoints"] From 542b83ef9d0818580a447c5d3791abd96a5fd7fa Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 4 Jul 2023 14:40:27 +0200 Subject: [PATCH 09/45] fix bug with non-visible keypoints - breaks CLI --- keypoint_detection/data/coco_dataset.py | 26 +++++++++++++++---------- test/configuration.py | 2 +- test/test_dataset.py | 3 ++- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/keypoint_detection/data/coco_dataset.py b/keypoint_detection/data/coco_dataset.py index c006a97..04be705 100644 --- a/keypoint_detection/data/coco_dataset.py +++ b/keypoint_detection/data/coco_dataset.py @@ -42,18 +42,20 @@ def add_argparse_args(parent_parser: argparse.ArgumentParser) -> argparse.Argume """ parser = parent_parser.add_argument_group("COCOkeypointsDataset") parser.add_argument( - "--detect_non_visible_keypoints", - default=True, - type=str, - help="detect keypoints with visibility flag = 1? default = True", + "--detect_only_visible_keypoints", + dest="detect_only_visible_keypoints", + default=False, + action="store_true", + help="If set, only keypoints with flag > 1.0 will be used.", ) + return parent_parser def __init__( self, json_dataset_path: str, keypoint_channel_configuration: list[list[str]], - detect_non_visible_keypoints: bool = True, + detect_only_visible_keypoints: bool = True, transform: A.Compose = None, imageloader: ImageLoader = None, **kwargs, @@ -65,7 +67,9 @@ def __init__( self.dataset_dir_path = self.dataset_json_path.parent # assume paths in JSON are relative to this directory! self.keypoint_channel_configuration = keypoint_channel_configuration - self.detect_non_visible_keypoints = detect_non_visible_keypoints + self.detect_only_visible_keypoints = detect_only_visible_keypoints + + print(f"{detect_only_visible_keypoints=}") self.random_crop_transform = None self.transform = transform @@ -171,10 +175,12 @@ def is_keypoint_visible(self, keypoint: COCO_KEYPOINT_TYPE) -> bool: Returns: bool: True if current keypoint is considered visible according to the dataset configuration, else False """ - minimal_flag = 0 - if not self.detect_non_visible_keypoints: - minimal_flag = 1 - return keypoint[2] > minimal_flag + if self.detect_only_visible_keypoints: + # filter out occluded keypoints with flag 1.0 + return keypoint[2] > 1.5 + else: + # filter out non-labeled keypoints with flag 0.0 + return keypoint[2] > 0.5 @staticmethod def split_list_in_keypoints(list_to_split: List[COCO_KEYPOINT_TYPE]) -> List[List[COCO_KEYPOINT_TYPE]]: diff --git a/test/configuration.py b/test/configuration.py index c4f5570..72f267e 100644 --- a/test/configuration.py +++ b/test/configuration.py @@ -11,7 +11,7 @@ DEFAULT_HPARAMS = { "keypoint_channel_configuration": [["box_corner0", "box_corner1", "box_corner2", "box_corner3"], ["flap_corner0"]], - "detect_non_visible_keypoints": True, + "detect_only_visible_keypoints": False, "seed": 102, "wandb_project": "test_project", "wandb_entity": "box-manipulation", diff --git a/test/test_dataset.py b/test/test_dataset.py index 4c641c5..8538f0e 100644 --- a/test/test_dataset.py +++ b/test/test_dataset.py @@ -41,7 +41,8 @@ def test_dataset(self): self.assertEqual(len(ch2), len(DEFAULT_HPARAMS["keypoint_channel_configuration"][1])) def test_non_visible_dataset(self): - self.hparams.update({"detect_non_visible_keypoints": False}) + self.hparams["json_dataset_path"] = Path(__file__).parent / "test_dataset" / "duplicate_coco_dataset.json" + self.hparams.update({"detect_only_visible_keypoints": True}) dataset = COCOKeypointsDataset(**self.hparams) # has duplicates but they are not visible (flag=1) From 2f8e0a5d73fe326f5e1307949bfc07da4a072557 Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 4 Jul 2023 14:44:26 +0200 Subject: [PATCH 10/45] remove worker seeding in favor of PL functionality --- keypoint_detection/data/datamodule.py | 27 ++++++++++++++++----------- keypoint_detection/train/train.py | 7 +++++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/keypoint_detection/data/datamodule.py b/keypoint_detection/data/datamodule.py index 333ef48..029c28b 100644 --- a/keypoint_detection/data/datamodule.py +++ b/keypoint_detection/data/datamodule.py @@ -106,33 +106,32 @@ def _split_dataset(dataset, validation_split_ratio): return train_dataset, validation_dataset def train_dataloader(self): + # usually need to seed workers for reproducibility + # cf. https://pytorch.org/docs/stable/notes/randomness.html + # but PL does for us in their seeding function: + # https://lightning.ai/docs/pytorch/stable/common/trainer.html#reproducibility + dataloader = DataLoader( self.train_dataset, self.batch_size, shuffle=True, num_workers=self.num_workers, collate_fn=COCOKeypointsDataset.collate_fn, - pin_memory=True, + pin_memory=True, # usually a little faster ) return dataloader def val_dataloader(self): - def seed_worker(worker_id): - worker_seed = torch.initial_seed() % 2**32 - np.random.seed(worker_seed) - random.seed(worker_seed) - - g = torch.Generator() - g.manual_seed(0) - # num workers to zero to avoid non-reproducibility bc of random seeds for workers + # usually need to seed workers for reproducibility # cf. https://pytorch.org/docs/stable/notes/randomness.html + # but PL does for us in their seeding function: + # https://lightning.ai/docs/pytorch/stable/common/trainer.html#reproducibility + dataloader = DataLoader( self.validation_dataset, self.batch_size, shuffle=False, num_workers=self.num_workers, - worker_init_fn=seed_worker, - generator=g, collate_fn=COCOKeypointsDataset.collate_fn, ) return dataloader @@ -146,3 +145,9 @@ def test_dataloader(self): collate_fn=COCOKeypointsDataset.collate_fn, ) return dataloader + + +def seed_worker(worker_id): + worker_seed = torch.initial_seed() % 2**32 + np.random.seed(worker_seed) + random.seed(worker_seed) \ No newline at end of file diff --git a/keypoint_detection/train/train.py b/keypoint_detection/train/train.py index e6ab3cc..f22c75c 100644 --- a/keypoint_detection/train/train.py +++ b/keypoint_detection/train/train.py @@ -45,7 +45,14 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: Initializes the datamodule, model and trainer based on the global hyperparameters. calls trainer.fit(model, module) afterwards and returns both model and trainer. """ + # seed all random number generators on all processes and workers for reproducibility pl.seed_everything(hparams["seed"], workers=True) + # you can uncomment the following lines to make training more reproducible + # but the impact is limited in my experience. + # see https://pytorch.org/docs/stable/notes/randomness.html#reproducibility + # import torch + # torch.backends.cudnn.deterministic = True + # torch.backends.cudnn.benchmark = False backbone = BackboneFactory.create_backbone(**hparams) model = KeypointDetector(backbone=backbone, **hparams) From e2dee52a2c2515d69770db11157f3ea33eab9dc5 Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 4 Jul 2023 14:45:49 +0200 Subject: [PATCH 11/45] format --- keypoint_detection/data/datamodule.py | 8 ++++---- keypoint_detection/train/train.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/keypoint_detection/data/datamodule.py b/keypoint_detection/data/datamodule.py index 029c28b..6388d64 100644 --- a/keypoint_detection/data/datamodule.py +++ b/keypoint_detection/data/datamodule.py @@ -106,7 +106,7 @@ def _split_dataset(dataset, validation_split_ratio): return train_dataset, validation_dataset def train_dataloader(self): - # usually need to seed workers for reproducibility + # usually need to seed workers for reproducibility # cf. https://pytorch.org/docs/stable/notes/randomness.html # but PL does for us in their seeding function: # https://lightning.ai/docs/pytorch/stable/common/trainer.html#reproducibility @@ -117,12 +117,12 @@ def train_dataloader(self): shuffle=True, num_workers=self.num_workers, collate_fn=COCOKeypointsDataset.collate_fn, - pin_memory=True, # usually a little faster + pin_memory=True, # usually a little faster ) return dataloader def val_dataloader(self): - # usually need to seed workers for reproducibility + # usually need to seed workers for reproducibility # cf. https://pytorch.org/docs/stable/notes/randomness.html # but PL does for us in their seeding function: # https://lightning.ai/docs/pytorch/stable/common/trainer.html#reproducibility @@ -150,4 +150,4 @@ def test_dataloader(self): def seed_worker(worker_id): worker_seed = torch.initial_seed() % 2**32 np.random.seed(worker_seed) - random.seed(worker_seed) \ No newline at end of file + random.seed(worker_seed) diff --git a/keypoint_detection/train/train.py b/keypoint_detection/train/train.py index f22c75c..aafb0ca 100644 --- a/keypoint_detection/train/train.py +++ b/keypoint_detection/train/train.py @@ -50,7 +50,7 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: # you can uncomment the following lines to make training more reproducible # but the impact is limited in my experience. # see https://pytorch.org/docs/stable/notes/randomness.html#reproducibility - # import torch + # import torch # torch.backends.cudnn.deterministic = True # torch.backends.cudnn.benchmark = False From 09e1476ccc1a5fb69c78b55c7cc0c6f1915d9e57 Mon Sep 17 00:00:00 2001 From: tlpss Date: Mon, 14 Aug 2023 14:44:38 +0200 Subject: [PATCH 12/45] fix image size bug for augmentations --- keypoint_detection/data/datamodule.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/keypoint_detection/data/datamodule.py b/keypoint_detection/data/datamodule.py index 6388d64..302dae1 100644 --- a/keypoint_detection/data/datamodule.py +++ b/keypoint_detection/data/datamodule.py @@ -81,13 +81,14 @@ def __init__( # create the transforms if needed and set them to the datasets if augment_train: - img_width, img_height = self.train_dataset[0][0].shape[1], self.train_dataset[0][0].shape[2] + print("Augmenting the training dataset!") + img_height, img_width = self.train_dataset[0][0].shape[1], self.train_dataset[0][0].shape[2] train_transform = MultiChannelKeypointsCompose( [ - A.ColorJitter(), - A.RandomRotate90(), - A.HorizontalFlip(), - A.RandomResizedCrop(img_height, img_width, scale=(0.8, 1.0), ratio=(0.95, 1.0)), + A.ColorJitter(p=1.0), + A.RandomBrightnessContrast(p=1.0), + # A.RandomRotate90(), + A.RandomResizedCrop(img_height, img_width, scale=(0.8, 1.0), ratio=(0.95, 1.0), p=0.7), ] ) if isinstance(self.train_dataset, COCOKeypointsDataset): From 3c2c68f363b2b9ce0289490d15d2b3c76ce303a7 Mon Sep 17 00:00:00 2001 From: Thomas Lips <37955681+tlpss@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:11:53 +0200 Subject: [PATCH 13/45] make detector fully deterministic by default (#30) * make pytorch use deterministic algorithms * change all upsample layers from bilinear to NN to make them determinstic --- .../models/backbones/convnext_unet.py | 12 +++---- .../models/backbones/maxvit_unet.py | 8 ++--- .../models/backbones/mobilenetv3.py | 8 ++--- keypoint_detection/models/backbones/unet.py | 3 +- keypoint_detection/train/train.py | 31 ++++++++++++++++--- 5 files changed, 41 insertions(+), 21 deletions(-) diff --git a/keypoint_detection/models/backbones/convnext_unet.py b/keypoint_detection/models/backbones/convnext_unet.py index f9b4fa3..1794852 100644 --- a/keypoint_detection/models/backbones/convnext_unet.py +++ b/keypoint_detection/models/backbones/convnext_unet.py @@ -24,7 +24,8 @@ class UpSamplingBlock(nn.Module): def __init__(self, n_channels_in, n_skip_channels_in, n_channels_out, kernel_size): super().__init__() - self.upsample = nn.UpsamplingBilinear2d(scale_factor=2) + # bilinear is not deterministic, use nearest neighbor instead + self.upsample = lambda x: nn.functional.interpolate(x, scale_factor=2) self.conv_reduce = nn.Conv2d( in_channels=n_channels_in, out_channels=n_skip_channels_in, kernel_size=1, bias=False, padding="same" ) @@ -61,7 +62,7 @@ class ConvNeXtUnet(Backbone): (head) - stem final_up (bilinear 4x) + stem final_up ( 4x) res1 ---> 1/4 decode3 res2 ---> 1/8 decode2 res3 ---> 1/16 decode1 @@ -82,9 +83,7 @@ def __init__(self, **kwargs): block = UpSamplingBlock(channels_in, skip_channels_in, skip_channels_in, 3) self.decoder_blocks.append(block) - self.final_upsampling_block = nn.Sequential( - nn.UpsamplingBilinear2d(scale_factor=4), nn.Conv2d(skip_channels_in, skip_channels_in, 3, padding="same") - ) + self.final_conv = nn.Conv2d(skip_channels_in, skip_channels_in, 3, padding="same") def forward(self, x): features = self.encoder(x) @@ -92,7 +91,8 @@ def forward(self, x): x = features.pop() for block in self.decoder_blocks: x = block(x, features.pop()) - x = self.final_upsampling_block(x) + x = nn.functional.interpolate(x, scale_factor=4) + x = self.final_conv(x) return x def get_n_channels_out(self): diff --git a/keypoint_detection/models/backbones/maxvit_unet.py b/keypoint_detection/models/backbones/maxvit_unet.py index fe4173d..140ab84 100644 --- a/keypoint_detection/models/backbones/maxvit_unet.py +++ b/keypoint_detection/models/backbones/maxvit_unet.py @@ -55,9 +55,8 @@ def __init__(self, **kwargs) -> None: block = UpSamplingBlock(config_in["channels"], config_skip["channels"], config_skip["channels"], 3) self.decoder_blocks.append(block) - self.final_upsampling_block = nn.Sequential( - nn.UpsamplingBilinear2d(scale_factor=2), - nn.Conv2d(self.feature_config[0]["channels"], self.feature_config[0]["channels"], 3, padding="same"), + self.final_conv = nn.Conv2d( + self.feature_config[0]["channels"], self.feature_config[0]["channels"], 3, padding="same" ) def forward(self, x): @@ -65,7 +64,8 @@ def forward(self, x): x = features.pop(-1) for block in self.decoder_blocks[::-1]: x = block(x, features.pop(-1)) - x = self.final_upsampling_block(x) + x = nn.functional.interpolate(x, scale_factor=2) + x = self.final_conv(x) return x def get_n_channels_out(self): diff --git a/keypoint_detection/models/backbones/mobilenetv3.py b/keypoint_detection/models/backbones/mobilenetv3.py index d498727..fe830ca 100644 --- a/keypoint_detection/models/backbones/mobilenetv3.py +++ b/keypoint_detection/models/backbones/mobilenetv3.py @@ -25,10 +25,7 @@ def __init__(self, **kwargs): block = UpSamplingBlock(channels_in, skip_channels_in, skip_channels_in, 3) self.decoder_blocks.append(block) - self.final_upsampling_block = nn.Sequential( - nn.UpsamplingBilinear2d(scale_factor=2), - nn.Conv2d(skip_channels_in, skip_channels_in, 3, padding="same"), - ) + self.final_conv = nn.Conv2d(skip_channels_in, skip_channels_in, 3, padding="same") def forward(self, x: torch.Tensor) -> torch.Tensor: features = self.encoder(x) @@ -36,7 +33,8 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: x = features.pop() for block in self.decoder_blocks: x = block(x, features.pop()) - x = self.final_upsampling_block(x) + x = nn.functional.interpolate(x, scale_factor=2) + x = self.final_conv(x) return x diff --git a/keypoint_detection/models/backbones/unet.py b/keypoint_detection/models/backbones/unet.py index 48581f8..41b0f1e 100644 --- a/keypoint_detection/models/backbones/unet.py +++ b/keypoint_detection/models/backbones/unet.py @@ -43,7 +43,6 @@ def forward(self, x): class UpSamplingBlock(nn.Module): def __init__(self, n_channels_in, n_channels_out, kernel_size): super().__init__() - self.upsample = nn.UpsamplingBilinear2d(scale_factor=2) self.conv = nn.Conv2d( in_channels=n_channels_in * 2, out_channels=n_channels_out, @@ -55,7 +54,7 @@ def __init__(self, n_channels_in, n_channels_out, kernel_size): self.relu = nn.ReLU() def forward(self, x, x_skip): - x = self.upsample(x) + x = nn.functional.interpolate(x, scale_factor=2) x = torch.cat([x, x_skip], dim=1) x = self.conv(x) x = self.relu(x) diff --git a/keypoint_detection/train/train.py b/keypoint_detection/train/train.py index aafb0ca..2448859 100644 --- a/keypoint_detection/train/train.py +++ b/keypoint_detection/train/train.py @@ -37,6 +37,18 @@ def add_system_args(parent_parser: ArgumentParser) -> ArgumentParser: type=float, help="relative threshold for early stopping callback. If validation epoch loss does not increase with at least this fraction compared to the best result so far for 5 consecutive epochs, training is stopped.", ) + # deterministic argument for PL trainer, not exposed in their CLI. + # https://lightning.ai/docs/pytorch/stable/common/trainer.html#reproducibility + # set to True by default, but can be set to False to speed up training. + + parser.add_argument( + "--non-deterministic-pytorch", + action="store_false", + dest="deterministic", + help="do not use deterministic algorithms for pytorch. This can speed up training, but will make it non-reproducible.", + ) + + parser.set_defaults(deterministic=True) return parent_parser @@ -47,12 +59,23 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: """ # seed all random number generators on all processes and workers for reproducibility pl.seed_everything(hparams["seed"], workers=True) - # you can uncomment the following lines to make training more reproducible + + # use deterministic algorithms for torch to ensure exact reproducibility + # https://pytorch.org/docs/stable/notes/randomness.html#reproducibility + # this can slow down training # but the impact is limited in my experience. - # see https://pytorch.org/docs/stable/notes/randomness.html#reproducibility - # import torch + # so I prefer to be deterministic (and hence reproducible) by default. + + # also note that following is not enough: # torch.backends.cudnn.deterministic = True - # torch.backends.cudnn.benchmark = False + # there are other non-deterministic algorithms + # cf list at https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html#torch.use_deterministic_algorithms + + # the following is still not good enough with Pytorch-Lightning: + # import torch + # torch.use_deterministic_algorithms(True) + # though I am not exactly sure why. + # so we have to set it in the trainer! (see create_pl_trainer) backbone = BackboneFactory.create_backbone(**hparams) model = KeypointDetector(backbone=backbone, **hparams) From 2a754118114a22b2b1db7ef793e059e82da10c32 Mon Sep 17 00:00:00 2001 From: tlpss Date: Mon, 21 Aug 2023 17:13:14 +0200 Subject: [PATCH 14/45] add citation guidelines Fixes Create citation #29 --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 87eed6e..56cb60b 100644 --- a/README.md +++ b/README.md @@ -74,3 +74,15 @@ TODO - why this repo? - why not label keypoints as bboxes and use YOLO/Detectron2? - .. + +# Citing this project + +You are invited to cite the following publication if you use this keypoint detector in your research: +``` +@inproceedings{lips2022synthkeypoints, + title={Learning Keypoints from Synthetic Data for Robotic Cloth Folding}, + author={Lips, Thomas and De Gusseme, Victor-Louis and others}, + journal={2nd workshop on Representing and Manipulating Deformable Objects - ICRA}, + year={2022} +} +``` From f63daca07b059081ef05385b491c261c74ad9843 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 25 Aug 2023 09:14:26 +0200 Subject: [PATCH 15/45] add some print statements to dataset splitting --- keypoint_detection/data/datamodule.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/keypoint_detection/data/datamodule.py b/keypoint_detection/data/datamodule.py index 302dae1..acd53f7 100644 --- a/keypoint_detection/data/datamodule.py +++ b/keypoint_detection/data/datamodule.py @@ -55,7 +55,7 @@ def __init__( json_validation_dataset_path: str = None, json_test_dataset_path=None, augment_train: bool = False, - **kwargs + **kwargs, ): super().__init__() self.batch_size = batch_size @@ -72,6 +72,7 @@ def __init__( json_validation_dataset_path, keypoint_channel_configuration, **kwargs ) else: + print(f"splitting the train set to create a validation set with ratio {validation_split_ratio} ") self.train_dataset, self.validation_dataset = KeypointsDataModule._split_dataset( self.train_dataset, validation_split_ratio ) @@ -104,6 +105,8 @@ def _split_dataset(dataset, validation_split_ratio): validation_size = int(validation_split_ratio * len(dataset)) train_size = len(dataset) - validation_size train_dataset, validation_dataset = torch.utils.data.random_split(dataset, [train_size, validation_size]) + print(f"train size: {len(train_dataset)}") + print(f"validation size: {len(validation_dataset)}") return train_dataset, validation_dataset def train_dataloader(self): From 79d29845602f90373bee31a59cf9f38c6d8efa22 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 25 Aug 2023 09:55:30 +0200 Subject: [PATCH 16/45] remove duplicate brightness&contrast transform --- keypoint_detection/data/datamodule.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/keypoint_detection/data/datamodule.py b/keypoint_detection/data/datamodule.py index acd53f7..30b086c 100644 --- a/keypoint_detection/data/datamodule.py +++ b/keypoint_detection/data/datamodule.py @@ -86,10 +86,8 @@ def __init__( img_height, img_width = self.train_dataset[0][0].shape[1], self.train_dataset[0][0].shape[2] train_transform = MultiChannelKeypointsCompose( [ - A.ColorJitter(p=1.0), - A.RandomBrightnessContrast(p=1.0), - # A.RandomRotate90(), - A.RandomResizedCrop(img_height, img_width, scale=(0.8, 1.0), ratio=(0.95, 1.0), p=0.7), + A.ColorJitter(p=0.8), + A.RandomResizedCrop(img_height, img_width, scale=(0.8, 1.0), ratio=(0.9, 1.1), p=1.0), ] ) if isinstance(self.train_dataset, COCOKeypointsDataset): From 20c3f51d878540ac6e69821ad55b3ccfe2eb3bad Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 25 Aug 2023 10:21:23 +0200 Subject: [PATCH 17/45] use best ckpt for test set evaluation --- keypoint_detection/train/train.py | 10 +++++++++- keypoint_detection/train/utils.py | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/keypoint_detection/train/train.py b/keypoint_detection/train/train.py index 2448859..495a60a 100644 --- a/keypoint_detection/train/train.py +++ b/keypoint_detection/train/train.py @@ -90,7 +90,15 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: trainer.fit(model, data_module) if "json_test_dataset_path" in hparams: - trainer.test(model, data_module) + # check if we have a best checkpoint, if not, use the current weights but log a warning + # it makes more sense to evaluate on the best checkpoint because, i.e. the best validation score obtained. + # evaluating on the current weights is more noisy and would also result in lower evaluation scores if overfitting happens + # when training longer, even with perfect i.i.d. test/val sets. This is not desired. + + ckpt_path = trainer.checkpoint_callback.best_model_path + if ckpt_path == "": + print("No best checkpoint found, using current weights for test set evaluation") + trainer.test(model, data_module, ckpt_path="best") return model, trainer diff --git a/keypoint_detection/train/utils.py b/keypoint_detection/train/utils.py index 22ea20c..5e62718 100644 --- a/keypoint_detection/train/utils.py +++ b/keypoint_detection/train/utils.py @@ -82,6 +82,11 @@ def create_pl_trainer(hparams: dict, wandb_logger: WandbLogger) -> Trainer: ) # cf https://pytorch-lightning.readthedocs.io/en/latest/api/pytorch_lightning.loggers.wandb.html + # would be better to use mAP metric for checkpointing, but this is not calculated every epoch because it is rather expensive + # (and actually this is due to the keypoint extraction from the heatmaps..) + # TODO: make this extraction faster by doing it on GPU? + + # epoch_loss still correlates rather well though checkpoint_callback = ModelCheckpoint(monitor="validation/epoch_loss", mode="min") trainer = pl.Trainer(**trainer_kwargs, callbacks=[early_stopping, checkpoint_callback]) From 2c95d65d7ac5b6689f9cf67673642d75f850a239 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 25 Aug 2023 16:51:32 +0200 Subject: [PATCH 18/45] reduce amount of logged data --- keypoint_detection/models/detector.py | 25 ++++++++++++++++--------- keypoint_detection/train/train.py | 4 +++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index 3b10347..f09887d 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -256,6 +256,7 @@ def shared_step(self, batch, batch_idx, include_visualization_data_in_result_dic def training_step(self, train_batch, batch_idx): log_images = batch_idx == 0 and self.current_epoch > 0 + result_dict = self.shared_step(train_batch, batch_idx, include_visualization_data_in_result_dict=log_images) if log_images: @@ -321,10 +322,10 @@ def validation_step(self, val_batch, batch_idx): if self.is_ap_epoch(): self.update_ap_metrics(result_dict, self.ap_validation_metrics) - log_images = batch_idx == 0 and self.current_epoch > 0 - if log_images: - image_grids = self.visualize_predictions_channels(result_dict) - self.log_image_grids(image_grids, mode="validation") + log_images = batch_idx == 0 and self.current_epoch > 0 + if log_images: + image_grids = self.visualize_predictions_channels(result_dict) + self.log_image_grids(image_grids, mode="validation") ## log (defaults to on_epoch, which aggregates the logged values over entire validation set) self.log("validation/epoch_loss", result_dict["loss"]) @@ -334,8 +335,10 @@ def test_step(self, test_batch, batch_idx): # no need to switch model to eval mode, this is handled by pytorch lightning result_dict = self.shared_step(test_batch, batch_idx, include_visualization_data_in_result_dict=True) self.update_ap_metrics(result_dict, self.ap_test_metrics) - image_grids = self.visualize_predictions_channels(result_dict) - self.log_image_grids(image_grids, mode="test") + # only log first 10 batches to reduce storage space + if batch_idx < 10: + image_grids = self.visualize_predictions_channels(result_dict) + self.log_image_grids(image_grids, mode="test") self.log("test/epoch_loss", result_dict["loss"]) self.log("test/gt_loss", result_dict["gt_loss"]) @@ -405,9 +408,13 @@ def compute_and_log_metrics_for_channel( def is_ap_epoch(self) -> bool: """Returns True if the AP should be calculated in this epoch.""" - return ( - self.ap_epoch_start <= self.current_epoch and self.current_epoch % self.ap_epoch_freq == 0 - ) or self.current_epoch == self.trainer.max_epochs - 1 + is_epch = self.ap_epoch_start <= self.current_epoch and self.current_epoch % self.ap_epoch_freq == 0 + # always log the AP in the last epoch + is_epch = is_epch or self.current_epoch == self.trainer.max_epochs - 1 + + # if user manually specified a validation frequency, we should always log the AP in that epoch + is_epch = is_epch or (self.current_epoch > 0 and self.trainer.check_val_every_n_epoch > 1) + return is_epch def extract_detected_keypoints_from_heatmap(self, heatmap: torch.Tensor) -> List[DetectedKeypoint]: """ diff --git a/keypoint_detection/train/train.py b/keypoint_detection/train/train.py index 495a60a..9dfbe1d 100644 --- a/keypoint_detection/train/train.py +++ b/keypoint_detection/train/train.py @@ -84,7 +84,9 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: project=hparams["wandb_project"], entity=hparams["wandb_entity"], save_dir=get_wandb_log_dir_path(), - log_model="all", # log all checkpoints made by PL, see create_trainer for callback + log_model=True, # only log checkpoints at the end of training, i.e. only log the best checkpoint + # not suitable for expensive training runs where you might want to restart from checkpoint + # but this saves storage and usually keypoint detector training runs are not that expensive anyway ) trainer = create_pl_trainer(hparams, wandb_logger) trainer.fit(model, data_module) From 6379eaca5ecfa283ab1cd7f0ac51b60fb78274dc Mon Sep 17 00:00:00 2001 From: tlpss Date: Mon, 28 Aug 2023 08:31:52 +0200 Subject: [PATCH 19/45] add final upsampling block on original resolution and remove reducing conv in upsampling block --- .../models/backbones/convnext_unet.py | 38 +++++++------- .../models/backbones/maxvit_unet.py | 51 ++++++++++++------- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/keypoint_detection/models/backbones/convnext_unet.py b/keypoint_detection/models/backbones/convnext_unet.py index 1794852..5550232 100644 --- a/keypoint_detection/models/backbones/convnext_unet.py +++ b/keypoint_detection/models/backbones/convnext_unet.py @@ -26,27 +26,26 @@ def __init__(self, n_channels_in, n_skip_channels_in, n_channels_out, kernel_siz super().__init__() # bilinear is not deterministic, use nearest neighbor instead self.upsample = lambda x: nn.functional.interpolate(x, scale_factor=2) - self.conv_reduce = nn.Conv2d( - in_channels=n_channels_in, out_channels=n_skip_channels_in, kernel_size=1, bias=False, padding="same" - ) - self.conv = nn.Conv2d( - in_channels=n_skip_channels_in * 2, + self.conv1 = nn.Conv2d( + in_channels=n_skip_channels_in + n_channels_in, out_channels=n_channels_out, kernel_size=kernel_size, bias=False, padding="same", ) - self.norm = nn.BatchNorm2d(n_channels_out) - self.relu = nn.ReLU() + + self.norm1 = nn.BatchNorm2d(n_channels_out) + self.relu1 = nn.ReLU() def forward(self, x, x_skip): x = self.upsample(x) - x = self.conv_reduce(x) x = torch.cat([x, x_skip], dim=1) - x = self.conv(x) - x = self.norm(x) - x = self.relu(x) + x = self.conv1(x) + x = self.norm1(x) + x = self.relu1(x) + # second conv as in original UNet upsampling block decreases performance + # probably because I was using a small dataset that did not have enough data to learn the extra parameters return x @@ -61,12 +60,13 @@ class ConvNeXtUnet(Backbone): nano -> 17M params (but only twice as slow) - (head) - stem final_up ( 4x) - res1 ---> 1/4 decode3 - res2 ---> 1/8 decode2 - res3 ---> 1/16 decode1 - res4 ---1/32----| + input final_conv --- head + stem upsampling + upsamping + res1 ---> 1/4 decode3 + res2 ---> 1/8 decode2 + res3 ---> 1/16 decode1 + res4 ---1/32----| """ def __init__(self, **kwargs): @@ -83,15 +83,17 @@ def __init__(self, **kwargs): block = UpSamplingBlock(channels_in, skip_channels_in, skip_channels_in, 3) self.decoder_blocks.append(block) - self.final_conv = nn.Conv2d(skip_channels_in, skip_channels_in, 3, padding="same") + self.final_conv = nn.Conv2d(skip_channels_in + 3, skip_channels_in, 3, padding="same") def forward(self, x): + x_orig = torch.clone(x) features = self.encoder(x) x = features.pop() for block in self.decoder_blocks: x = block(x, features.pop()) x = nn.functional.interpolate(x, scale_factor=4) + x = torch.cat([x, x_orig], dim=1) x = self.final_conv(x) return x diff --git a/keypoint_detection/models/backbones/maxvit_unet.py b/keypoint_detection/models/backbones/maxvit_unet.py index 140ab84..fccdcf8 100644 --- a/keypoint_detection/models/backbones/maxvit_unet.py +++ b/keypoint_detection/models/backbones/maxvit_unet.py @@ -28,12 +28,13 @@ class MaxVitUnet(Backbone): For now only 256 is supported so input sizes are restricted to 256,512,... - (head) - stem --- 1/2 --> final_up (bilinear 2x) - stage 1 --- 1/4 --> decode3 - stage 2 --- 1/8 --> decode2 - stage 3 --- 1/16 --> decode1 - stage 4 ---1/32----| + + orig --- 1/1 --> ---> (head) + stem --- 1/2 --> decode4 + stage 1 --- 1/4 --> decode3 + stage 2 --- 1/8 --> decode2 + stage 3 --- 1/16 --> decode1 + stage 4 ---1/32----| """ # manually gathered for maxvit_nano_rw_256 @@ -58,14 +59,20 @@ def __init__(self, **kwargs) -> None: self.final_conv = nn.Conv2d( self.feature_config[0]["channels"], self.feature_config[0]["channels"], 3, padding="same" ) + self.final_upsampling_block = UpSamplingBlock( + self.feature_config[0]["channels"], 3, self.feature_config[0]["channels"], 3 + ) def forward(self, x): + orig_x = torch.clone(x) features = list(self.feature_extractor(x).values()) x = features.pop(-1) for block in self.decoder_blocks[::-1]: x = block(x, features.pop(-1)) - x = nn.functional.interpolate(x, scale_factor=2) - x = self.final_conv(x) + + # x = nn.functional.interpolate(x, scale_factor=2) + # x = self.final_conv(x) + x = self.final_upsampling_block(x, orig_x) return x def get_n_channels_out(self): @@ -74,15 +81,21 @@ def get_n_channels_out(self): if __name__ == "__main__": # model = timm.create_model("maxvit_rmlp_pico_rw_256") - model = timm.create_model("maxvit_nano_rw_256") - feature_extractor = create_feature_extractor(model, ["stem", "stages.0", "stages.1", "stages.2", "stages.3"]) + # model = timm.create_model("maxvit_nano_rw_256") + # feature_extractor = create_feature_extractor(model, ["stem", "stages.0", "stages.1", "stages.2", "stages.3"]) + # x = torch.zeros((1, 3, 256, 256)) + # features = list(feature_extractor(x).values()) + # n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + # print(f"num params = {n_params/10**6:.2f} M") + # feature_config = [] + # for x in features: + # print(f"{x.shape=}") + # config = {"down": 256 // x.shape[2], "channels": x.shape[1]} + # feature_config.append(config) + # print(f"{feature_config=}") + + print("creating MaxViTUnet") + model = MaxVitUnet() x = torch.zeros((1, 3, 256, 256)) - features = list(feature_extractor(x).values()) - n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) - print(f"num params = {n_params/10**6:.2f} M") - feature_config = [] - for x in features: - print(f"{x.shape=}") - config = {"down": 256 // x.shape[2], "channels": x.shape[1]} - feature_config.append(config) - print(f"{feature_config=}") + y = model(x) + print(f"{y.shape=}") From b6429e7cad9d758b986c2e05c58d5311890d048e Mon Sep 17 00:00:00 2001 From: Thomas Lips <37955681+tlpss@users.noreply.github.com> Date: Tue, 29 Aug 2023 12:43:04 +0200 Subject: [PATCH 20/45] Faster keypoint extraction on GPU with Pytorch (#31) 5x-70x speedup of keypoint extraction from heatmaps by switching from scipy.peak_local_max to a GPU implementation with Pytorch * faster heatmap extraction with pytorch * fix jit issue * faster heatmap extraction & testing/benchmarking * reduce logging and use new keypoint extraction method * fix bug in keypoint extraction --- .../models/backbones/convnext_unet.py | 8 +- keypoint_detection/models/detector.py | 54 +++++---- keypoint_detection/utils/heatmap.py | 109 +++++++++++++++++- keypoint_detection/utils/visualization.py | 19 +-- scripts/benchmark.py | 12 +- scripts/benchmark_heatmap_extraction.py | 52 +++++++++ scripts/checkpoint_inference.py | 6 +- test/test_heatmap.py | 28 ++++- 8 files changed, 228 insertions(+), 60 deletions(-) create mode 100644 scripts/benchmark_heatmap_extraction.py diff --git a/keypoint_detection/models/backbones/convnext_unet.py b/keypoint_detection/models/backbones/convnext_unet.py index 5550232..d454961 100644 --- a/keypoint_detection/models/backbones/convnext_unet.py +++ b/keypoint_detection/models/backbones/convnext_unet.py @@ -24,8 +24,7 @@ class UpSamplingBlock(nn.Module): def __init__(self, n_channels_in, n_skip_channels_in, n_channels_out, kernel_size): super().__init__() - # bilinear is not deterministic, use nearest neighbor instead - self.upsample = lambda x: nn.functional.interpolate(x, scale_factor=2) + self.conv1 = nn.Conv2d( in_channels=n_skip_channels_in + n_channels_in, out_channels=n_channels_out, @@ -38,7 +37,8 @@ def __init__(self, n_channels_in, n_skip_channels_in, n_channels_out, kernel_siz self.relu1 = nn.ReLU() def forward(self, x, x_skip): - x = self.upsample(x) + # bilinear is not deterministic, use nearest neighbor instead + x = nn.functional.interpolate(x, scale_factor=2.0) x = torch.cat([x, x_skip], dim=1) x = self.conv1(x) x = self.norm1(x) @@ -92,7 +92,7 @@ def forward(self, x): x = features.pop() for block in self.decoder_blocks: x = block(x, features.pop()) - x = nn.functional.interpolate(x, scale_factor=4) + x = nn.functional.interpolate(x, scale_factor=4.0) x = torch.cat([x, x_orig], dim=1) x = self.final_conv(x) return x diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index f09887d..748582c 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -9,13 +9,8 @@ from keypoint_detection.models.backbones.base_backbone import Backbone from keypoint_detection.models.metrics import DetectedKeypoint, Keypoint, KeypointAPMetrics -from keypoint_detection.utils.heatmap import ( - BCE_loss, - compute_keypoint_probability, - create_heatmap_batch, - get_keypoints_from_heatmap, -) -from keypoint_detection.utils.visualization import visualize_predictions +from keypoint_detection.utils.heatmap import BCE_loss, create_heatmap_batch, get_keypoints_from_heatmap_batch_maxpool +from keypoint_detection.utils.visualization import visualize_predicted_heatmaps class KeypointDetector(pl.LightningModule): @@ -286,11 +281,10 @@ def visualize_predictions_channels(self, result_dict): image_grids = [] for channel_idx in range(len(self.keypoint_channel_configuration)): - grid = visualize_predictions( + grid = visualize_predicted_heatmaps( input_images, predicted_heatmaps[:, channel_idx, :, :], gt_heatmaps[channel_idx].cpu(), - minimal_keypoint_pixel_distance=6, ) image_grids.append(grid) return image_grids @@ -312,7 +306,7 @@ def logging_label(channel_configuration, mode: str) -> str: def log_image_grids(self, image_grids, mode: str): for channel_configuration, grid in zip(self.keypoint_channel_configuration, image_grids): label = KeypointDetector.logging_label(channel_configuration, mode) - image_caption = "top: predicted heatmaps, middle: predicted keypoints, bottom: gt heatmap" + image_caption = "top: predicted heatmaps, bottom: gt heatmaps" self.logger.experiment.log({label: wandb.Image(grid, caption=image_caption)}) def validation_step(self, val_batch, batch_idx): @@ -371,21 +365,24 @@ def update_channel_ap_metrics( self, predicted_heatmaps: torch.Tensor, gt_keypoints: List[torch.Tensor], validation_metric: KeypointAPMetrics ): """ - Updates the AP metric for a batch of heatmaps and keypoins of a single channel. + Updates the AP metric for a batch of heatmaps and keypoins of a single channel (!) This is done by extracting the detected keypoints for each heatmap and combining them with the gt keypoints for the same frame, so that the confusion matrix can be determined together with the distance thresholds. - predicted_heatmaps: N x H x W tensor + predicted_heatmaps: N x H x W tensor with the batch of predicted heatmaps for a single channel gt_keypoints: List of size N, containing K_i x 2 tensors with the ground truth keypoints for the channel of that sample """ - # log corner keypoints to AP metrics, frame by frame + # log corner keypoints to AP metrics for all images in this batch formatted_gt_keypoints = [ [Keypoint(int(k[0]), int(k[1])) for k in frame_gt_keypoints] for frame_gt_keypoints in gt_keypoints ] - for i, predicted_heatmap in enumerate(torch.unbind(predicted_heatmaps, 0)): - detected_keypoints = self.extract_detected_keypoints_from_heatmap(predicted_heatmap) - validation_metric.update(detected_keypoints, formatted_gt_keypoints[i]) + batch_detected_channel_keypoints = self.extract_detected_keypoints_from_heatmap( + predicted_heatmaps.unsqueeze(1) + ) + batch_detected_channel_keypoints = [batch_detected_channel_keypoints[i][0] for i in range(len(gt_keypoints))] + for i, detected_channel_keypoints in enumerate(batch_detected_channel_keypoints): + validation_metric.update(detected_channel_keypoints, formatted_gt_keypoints[i]) def compute_and_log_metrics_for_channel( self, metrics: KeypointAPMetrics, channel: str, training_mode: str @@ -423,14 +420,27 @@ def extract_detected_keypoints_from_heatmap(self, heatmap: torch.Tensor) -> List Args: heatmap (torch.Tensor) : H x W tensor that represents a heatmap. """ - - detected_keypoints = get_keypoints_from_heatmap( - heatmap, self.minimal_keypoint_pixel_distance, self.max_keypoints + if heatmap.dtype == torch.float16: + # Maxpool_2d not implemented for FP16 apparently + heatmap_to_extract_from = heatmap.float() + else: + heatmap_to_extract_from = heatmap + + keypoints, scores = get_keypoints_from_heatmap_batch_maxpool( + heatmap_to_extract_from, self.max_keypoints, self.minimal_keypoint_pixel_distance, return_scores=True ) - keypoint_probabilities = compute_keypoint_probability(heatmap, detected_keypoints) detected_keypoints = [ - DetectedKeypoint(detected_keypoints[i][0], detected_keypoints[i][1], keypoint_probabilities[i]) - for i in range(len(detected_keypoints)) + [[] for _ in range(heatmap_to_extract_from.shape[1])] for _ in range(heatmap_to_extract_from.shape[0]) ] + for batch_idx in range(len(detected_keypoints)): + for channel_idx in range(len(detected_keypoints[batch_idx])): + for kp_idx in range(len(keypoints[batch_idx][channel_idx])): + detected_keypoints[batch_idx][channel_idx].append( + DetectedKeypoint( + keypoints[batch_idx][channel_idx][kp_idx][0], + keypoints[batch_idx][channel_idx][kp_idx][1], + scores[batch_idx][channel_idx][kp_idx], + ) + ) return detected_keypoints diff --git a/keypoint_detection/utils/heatmap.py b/keypoint_detection/utils/heatmap.py index 497fc96..3ed72ef 100644 --- a/keypoint_detection/utils/heatmap.py +++ b/keypoint_detection/utils/heatmap.py @@ -1,4 +1,5 @@ -from typing import List, Tuple +import warnings +from typing import List, Optional, Tuple import numpy as np import torch @@ -77,23 +78,26 @@ def generate_channel_heatmap( return heatmap -def get_keypoints_from_heatmap( +def get_keypoints_from_heatmap_scipy( heatmap: torch.Tensor, min_keypoint_pixel_distance: int, max_keypoints: int = 20 ) -> List[Tuple[int, int]]: """ Extracts at most 20 keypoints from a heatmap, where each keypoint is defined as being a local maximum within a 2D mask [ -min_pixel_distance, + pixel_distance]^2 cf https://scikit-image.org/docs/dev/api/skimage.feature.html#skimage.feature.peak_local_max + THIS IS SLOW! use get_keypoints_from_heatmap_batch_maxpool instead. + + Args: heatmap : heatmap image - min_keypoint_pixel_distance : The size of the local mask + min_keypoint_pixel_distance : The size of the local mask, serves as NMS max_keypoints: the amount of keypoints to determine from the heatmap, -1 to return all points. Defaults to 20 to limit computational burder for models that predict random keypoints in early stage of training. Returns: A list of 2D keypoints """ - + warnings.warn("get_keypoints_from_heatmap_scipy is slow! Use get_keypoints_from_heatmap_batch_maxpool instead.") np_heatmap = heatmap.cpu().numpy().astype(np.float32) # num_peaks and rel_threshold are set to limit computational burden when models do random predictions. @@ -109,6 +113,89 @@ def get_keypoints_from_heatmap( return keypoints[::, ::-1].tolist() # convert to (u,v) aka (col,row) coord frame from (row,col) +def get_keypoints_from_heatmap_batch_maxpool( + heatmap: torch.Tensor, + max_keypoints: int = 20, + min_keypoint_pixel_distance: int = 1, + abs_max_threshold: Optional[float] = None, + rel_max_threshold: Optional[float] = None, + return_scores: bool = False, +) -> List[List[List[Tuple[int, int]]]]: + """Fast extraction of keypoints from a batch of heatmaps using maxpooling. + + Inspired by mmdetection and CenterNet: + https://mmdetection.readthedocs.io/en/v2.13.0/_modules/mmdet/models/utils/gaussian_target.html + + Args: + heatmap (torch.Tensor): NxCxHxW heatmap batch + max_keypoints (int, optional): max number of keypoints to extract, lowering will result in faster execution times. Defaults to 20. + min_keypoint_pixel_distance (int, optional): _description_. Defaults to 1. + + Following thresholds can be used at inference time to select where you want to be on the AP curve. They should ofc. not be used for training + abs_max_threshold (Optional[float], optional): _description_. Defaults to None. + rel_max_threshold (Optional[float], optional): _description_. Defaults to None. + + Returns: + The extracted keypoints for each batch, channel and heatmap; and their scores + """ + + # TODO: maybe separate the thresholding into another function to make sure it is not used during training, where it should not be used? + + # TODO: ugly that the output can change based on a flag.. should always return scores and discard them when I don't need them... + + batch_size, n_channels, _, width = heatmap.shape + + # obtain max_keypoints local maxima for each channel (w/ maxpool) + + kernel = min_keypoint_pixel_distance * 2 + 1 + pad = min_keypoint_pixel_distance + # exclude border keypoints by padding with highest possible value + # bc the borders are more susceptible to noise and could result in false positives + padded_heatmap = torch.nn.functional.pad(heatmap, (pad, pad, pad, pad), mode="constant", value=1.0) + max_pooled_heatmap = torch.nn.functional.max_pool2d(padded_heatmap, kernel, stride=1, padding=0) + # if the value equals the original value, it is the local maximum + local_maxima = max_pooled_heatmap == heatmap + # all values to zero that are not local maxima + heatmap = heatmap * local_maxima + + # extract top-k from heatmap (may include non-local maxima if there are less peaks than max_keypoints) + scores, indices = torch.topk(heatmap.view(batch_size, n_channels, -1), max_keypoints, sorted=True) + indices = torch.stack([torch.div(indices, width, rounding_mode="floor"), indices % width], dim=-1) + # at this point either score > 0.0, in which case the index is a local maximum + # or score is 0.0, in which case topk returned non-maxima, which will be filtered out later. + + # remove top-k that are not local maxima and threshold (if required) + # thresholding shouldn't be done during training + + # moving them to CPU now to avoid multiple GPU-mem accesses! + indices = indices.detach().cpu().numpy() + scores = scores.detach().cpu().numpy() + filtered_indices = [[[] for _ in range(n_channels)] for _ in range(batch_size)] + filtered_scores = [[[] for _ in range(n_channels)] for _ in range(batch_size)] + # determine NMS threshold + threshold = 0.01 # make sure it is > 0 to filter out top-k that are not local maxima + if abs_max_threshold is not None: + threshold = max(threshold, abs_max_threshold) + if rel_max_threshold is not None: + threshold = max(threshold, rel_max_threshold * heatmap.max()) + + # have to do this manually as the number of maxima for each channel can be different + for batch_idx in range(batch_size): + for channel_idx in range(n_channels): + candidates = indices[batch_idx, channel_idx] + for candidate_idx in range(candidates.shape[0]): + + # these are filtered out directly. + if scores[batch_idx, channel_idx, candidate_idx] > threshold: + # convert to (u,v) + filtered_indices[batch_idx][channel_idx].append(candidates[candidate_idx][::-1].tolist()) + filtered_scores[batch_idx][channel_idx].append(scores[batch_idx, channel_idx, candidate_idx]) + if return_scores: + return filtered_indices, filtered_scores + else: + return filtered_indices + + def compute_keypoint_probability(heatmap: torch.Tensor, detected_keypoints: List[Tuple[int, int]]) -> List[float]: """Compute probability measure for each detected keypoint on the heatmap @@ -121,3 +208,17 @@ def compute_keypoint_probability(heatmap: torch.Tensor, detected_keypoints: List """ # note the order! (u,v) is how we write , but the heatmap has to be indexed (v,u) as it is H x W return [heatmap[k[1]][k[0]].item() for k in detected_keypoints] + + +if __name__ == "__main__": + import torch.profiler as profiler + + keypoints = torch.tensor([[150, 134], [64, 153]]).cuda() + heatmap = generate_channel_heatmap((1080, 1920), keypoints, 6, "cuda") + heatmap = heatmap.unsqueeze(0).unsqueeze(0).repeat(1, 1, 1, 1) + # heatmap = torch.stack([heatmap, heatmap], dim=0) + print(heatmap.shape) + with profiler.profile(record_shapes=True) as prof: + with profiler.record_function("get_keypoints_from_heatmap_batch_maxpool"): + print(get_keypoints_from_heatmap_batch_maxpool(heatmap, 50, min_keypoint_pixel_distance=5)) + print(prof.key_averages().table(sort_by="cpu_time_total", row_limit=10)) diff --git a/keypoint_detection/utils/visualization.py b/keypoint_detection/utils/visualization.py index e2f0d39..d332f38 100644 --- a/keypoint_detection/utils/visualization.py +++ b/keypoint_detection/utils/visualization.py @@ -5,7 +5,7 @@ import torchvision from matplotlib import cm -from keypoint_detection.utils.heatmap import generate_channel_heatmap, get_keypoints_from_heatmap +from keypoint_detection.utils.heatmap import generate_channel_heatmap def overlay_image_with_heatmap(images: torch.Tensor, heatmaps: torch.Tensor, alpha=0.5) -> torch.Tensor: @@ -49,26 +49,17 @@ def overlay_image_with_keypoints(images: torch.Tensor, keypoints: List[torch.Ten return overlayed_images -def visualize_predictions( +def visualize_predicted_heatmaps( imgs: torch.Tensor, predicted_heatmaps: torch.Tensor, gt_heatmaps: torch.Tensor, - minimal_keypoint_pixel_distance: int, ): num_images = min(predicted_heatmaps.shape[0], 6) - keypoint_sigma = max(1, imgs.shape[2] / 64) predicted_heatmap_overlays = overlay_image_with_heatmap(imgs[:num_images], predicted_heatmaps[:num_images]) gt_heatmap_overlays = overlay_image_with_heatmap(imgs[:num_images], gt_heatmaps[:num_images]) - predicted_keypoints = [ - torch.tensor(get_keypoints_from_heatmap(predicted_heatmaps[i].cpu(), minimal_keypoint_pixel_distance)) - for i in range(predicted_heatmaps.shape[0]) - ] - predicted_keypoints_overlays = overlay_image_with_keypoints( - imgs[:num_images], predicted_keypoints[:num_images], keypoint_sigma - ) - - images = torch.cat([predicted_heatmap_overlays, predicted_keypoints_overlays, gt_heatmap_overlays]) + + images = torch.cat([predicted_heatmap_overlays, gt_heatmap_overlays]) grid = torchvision.utils.make_grid(images, nrow=num_images) return grid @@ -98,7 +89,7 @@ def visualize_predictions( shape = images.shape[2:] heatmaps = create_heatmap_batch(shape, keypoint_channels[0], sigma=6.0, device="cpu") - grid = visualize_predictions(images, heatmaps, heatmaps, 6) + grid = visualize_predicted_heatmaps(images, heatmaps, heatmaps, 6) image_numpy = grid.permute(1, 2, 0).numpy() plt.imshow(image_numpy) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index e0e5643..cc8b859 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -39,10 +39,10 @@ def benchmark(f, name=None, iters=500, warmup=20, display=True, profile=False): device = "cuda:0" backbone = "ConvNeXtUnet" - input_size = 256 + input_size = 512 backbone = BackboneFactory.create_backbone(backbone) - model = KeypointDetector(1, "2 4", 3, 3e-4, backbone, [["test"]], 1, 1, 0.0, 20) + model = KeypointDetector(1, "2 4", 3, 3e-4, backbone, [["test1"], ["test2,test3"]], 1, 1, 0.0, 20) # do not forget to set model to eval mode! # this will e.g. use the running statistics for batch norm layers instead of the batch statistics. # this is important as inference batches are typically a lot smaller which would create too much noise. @@ -52,14 +52,14 @@ def benchmark(f, name=None, iters=500, warmup=20, display=True, profile=False): sample_model_input = torch.rand(1, 3, input_size, input_size, device=device, dtype=torch.float32) sample_inference_input = np.random.randint(0, 255, (input_size, input_size, 3), dtype=np.uint8) - benchmark(lambda: model(sample_model_input), "plain model forward pass", profile=True) + benchmark(lambda: model(sample_model_input), "plain model forward pass", profile=False) benchmark( - lambda: local_inference(model, sample_inference_input, device=device), "plain model inference", profile=True + lambda: local_inference(model, sample_inference_input, device=device), "plain model inference", profile=False ) torchscript_model = model.to_torchscript() # JIT compiling with torchscript should improve performance (slightly) - benchmark(lambda: torchscript_model(sample_model_input), "torchscript model forward pass", profile=True) + benchmark(lambda: torchscript_model(sample_model_input), "torchscript model forward pass", profile=False) torch.backends.cudnn.benchmark = True model.half() @@ -67,7 +67,7 @@ def benchmark(f, name=None, iters=500, warmup=20, display=True, profile=False): half_torchscript_model = model.to_torchscript(method="trace", example_inputs=half_input) benchmark( - lambda: half_torchscript_model(half_input), "torchscript model forward pass with half precision", profile=True + lambda: half_torchscript_model(half_input), "torchscript model forward pass with half precision", profile=False ) # note: from the traces it can be seen that a lot of time is spent in 'overhead', i.e. the GPU is idle... diff --git a/scripts/benchmark_heatmap_extraction.py b/scripts/benchmark_heatmap_extraction.py new file mode 100644 index 0000000..c430b8f --- /dev/null +++ b/scripts/benchmark_heatmap_extraction.py @@ -0,0 +1,52 @@ +"""quick and dirty benchmark of the heatmap extraction methods.""" + +import time + +import torch + +from keypoint_detection.utils.heatmap import ( + generate_channel_heatmap, + get_keypoints_from_heatmap_batch_maxpool, + get_keypoints_from_heatmap_scipy, +) + + +def test_method(nb_iters, heatmaps, method, name): + n_keypoints = 20 + torch.cuda.synchronize() + t0 = time.time() + if method == get_keypoints_from_heatmap_scipy: + for i in range(nb_iters): + heatmap = heatmaps[i] + for batch in range(len(heatmap)): + for channel in range(len(heatmap[batch])): + method(heatmap[batch][channel], n_keypoints) + else: + for i in range(nb_iters): + method(heatmaps[i], n_keypoints) + torch.cuda.synchronize() + t1 = time.time() + duration = (t1 - t0) / nb_iters * 1000.0 + print(f"{duration:.3f} ms per iter for {name} method with heatmap size {heatmap_size} ") + + +if __name__ == "__main__": + nb_iters = 20 + n_channels = 2 + batch_size = 1 + n_keypoints_per_channel = 10 + print( + f"benchmarking with batch_size: {batch_size}, {n_channels} channels and {n_keypoints_per_channel} keypoints per channel" + ) + for heatmap_size in [(256, 256), (512, 256), (512, 512), (1920, 1080)]: + heatmaps = [ + generate_channel_heatmap(heatmap_size, torch.randint(0, 255, (6, 2)), 6, "cpu") + .unsqueeze(0) + .unsqueeze(0) + .repeat(batch_size, n_channels, 1, 1) + .cuda() + for _ in range(nb_iters) + ] + + test_method(nb_iters, heatmaps, get_keypoints_from_heatmap_scipy, "scipy") + test_method(nb_iters, heatmaps, get_keypoints_from_heatmap_batch_maxpool, "torch") diff --git a/scripts/checkpoint_inference.py b/scripts/checkpoint_inference.py index 426ccf8..bc10abb 100644 --- a/scripts/checkpoint_inference.py +++ b/scripts/checkpoint_inference.py @@ -4,7 +4,7 @@ import torch from torchvision.transforms.functional import to_tensor -from keypoint_detection.utils.heatmap import get_keypoints_from_heatmap +from keypoint_detection.utils.heatmap import get_keypoints_from_heatmap_batch_maxpool from keypoint_detection.utils.load_checkpoints import get_model_from_wandb_checkpoint @@ -27,9 +27,7 @@ def local_inference(model, image: np.ndarray, device="cuda"): heatmaps = model(image).squeeze(0) # extract keypoints from heatmaps - predicted_keypoints = [ - torch.tensor(get_keypoints_from_heatmap(heatmaps[i].cpu(), 2)) for i in range(heatmaps.shape[0]) - ] + predicted_keypoints = get_keypoints_from_heatmap_batch_maxpool(heatmaps.unsqueeze(0))[0] return predicted_keypoints diff --git a/test/test_heatmap.py b/test/test_heatmap.py index 585f5ae..903df27 100644 --- a/test/test_heatmap.py +++ b/test/test_heatmap.py @@ -3,7 +3,12 @@ import numpy as np import torch -from keypoint_detection.utils.heatmap import create_heatmap_batch, generate_channel_heatmap, get_keypoints_from_heatmap +from keypoint_detection.utils.heatmap import ( + create_heatmap_batch, + generate_channel_heatmap, + get_keypoints_from_heatmap_batch_maxpool, + get_keypoints_from_heatmap_scipy, +) class TestHeatmapUtils(unittest.TestCase): @@ -16,24 +21,35 @@ def setUp(self): def test_keypoint_generation_and_extraction(self): # test if extract(generate(keypoints)) == keypoints heatmap = generate_channel_heatmap((self.image_height, self.image_width), self.keypoints, self.sigma, "cpu") - extracted_keypoints = get_keypoints_from_heatmap(heatmap, 1) + extracted_keypoints = get_keypoints_from_heatmap_scipy(heatmap, 1) for keypoint in extracted_keypoints: self.assertTrue(keypoint in self.keypoints.tolist()) self.assertEqual((self.image_height, self.image_width), heatmap.shape) self.assertGreater(heatmap[4, 10], 0.5) - def test_extract_all_keypoints_from_heatmap(self): + def test_extract_all_keypoints_from_heatmap_scipy(self): def _test_extract_keypoints_from_heatmap(keypoints, num_keypoints): heatmap = generate_channel_heatmap((self.image_height, self.image_width), keypoints, self.sigma, "cpu") - extracted_keypoints = get_keypoints_from_heatmap(heatmap, 1, max_keypoints=num_keypoints) + extracted_keypoints = get_keypoints_from_heatmap_scipy(heatmap, 1, max_keypoints=num_keypoints) for keypoint in extracted_keypoints: self.assertTrue(keypoint in keypoints.tolist()) - keypoints = torch.randint(0, 15, (500, 2)) - _test_extract_keypoints_from_heatmap(keypoints, num_keypoints=500) + keypoints = torch.randint(0, 15, (5, 2)) + _test_extract_keypoints_from_heatmap(keypoints, num_keypoints=10) _test_extract_keypoints_from_heatmap(keypoints, num_keypoints=-1) _test_extract_keypoints_from_heatmap(keypoints, num_keypoints=np.inf) + def test_extract_keypoints_from_heatmap_maxpool(self): + def _test_extract_keypoints_from_heatmap(keypoints, num_keypoints): + heatmap = generate_channel_heatmap((self.image_height, self.image_width), keypoints, self.sigma, "cpu") + heatmap = heatmap.unsqueeze(0).unsqueeze(0) + extracted_keypoints = get_keypoints_from_heatmap_batch_maxpool(heatmap, max_keypoints=num_keypoints)[0][0] + for keypoint in extracted_keypoints: + self.assertTrue(keypoint in keypoints.tolist()) + + keypoints = torch.randint(0, 15, (5, 2)) + _test_extract_keypoints_from_heatmap(keypoints, num_keypoints=10) + def test_empty_heatmap(self): # test if heatmap for channel w/o keypoints is created correctly heatmap = generate_channel_heatmap( From e88c5f7285b3a20a867f48be7153e760076d93b5 Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 29 Aug 2023 17:37:59 +0200 Subject: [PATCH 21/45] only store model weights to further reduce storage --- keypoint_detection/train/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/keypoint_detection/train/utils.py b/keypoint_detection/train/utils.py index 5e62718..5ce8493 100644 --- a/keypoint_detection/train/utils.py +++ b/keypoint_detection/train/utils.py @@ -83,11 +83,13 @@ def create_pl_trainer(hparams: dict, wandb_logger: WandbLogger) -> Trainer: # cf https://pytorch-lightning.readthedocs.io/en/latest/api/pytorch_lightning.loggers.wandb.html # would be better to use mAP metric for checkpointing, but this is not calculated every epoch because it is rather expensive - # (and actually this is due to the keypoint extraction from the heatmaps..) - # TODO: make this extraction faster by doing it on GPU? - # epoch_loss still correlates rather well though - checkpoint_callback = ModelCheckpoint(monitor="validation/epoch_loss", mode="min") + # only store the best checkpoint and only the weights + # so cannot be used to resume training but only for inference + # saves storage though and training the detector is usually cheap enough to retrain it from scratch if you need specific weights etc. + checkpoint_callback = ModelCheckpoint( + monitor="validation/epoch_loss", mode="min", save_weights_only=True, save_top_k=1 + ) trainer = pl.Trainer(**trainer_kwargs, callbacks=[early_stopping, checkpoint_callback]) return trainer From 9f39a8e49524af27454e30311eb7b2a02339c8a6 Mon Sep 17 00:00:00 2001 From: Thomas Lips <37955681+tlpss@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:04:08 +0200 Subject: [PATCH 22/45] Fix metric for small distances (#32) There was a bug in the metrics that caused the heatmap generation and hence the metric to fail for d=1 if the keypoints were annotated as floats. This is now fixed by casting keypoints to the top-left (zero-index as in COCO) keypoint before training. make sure keypoints are cast to zero-indexed integers before training make sure heatmaps are centered on that pixel make sure the AP for d=0 (i.e pixel-perfect keypoints) goes up to 1 for a dummy dataset (integration testing) **These changes will break reproducibility** commits: * scrutinize the metric calculations: - all keypoints are in ints (and represent topleft corner of their pixel, aka zero-indexed) - heatmaps are really centered on the int pixel - metric max distances are ints and represent <= L2 distances (i.e. pixel-perfect == dmax = 0) - integration test: dummy dataset results in good scores for all mAP threholds down to 0 * remove deprecated test that is now broken * update readme * extend readme * extend readme --- .gitignore | 1 + README.md | 55 ++++++-- keypoint_detection/data/coco_dataset.py | 13 ++ keypoint_detection/data/coco_parser.py | 2 + keypoint_detection/models/detector.py | 4 +- keypoint_detection/models/metrics.py | 11 +- keypoint_detection/utils/heatmap.py | 4 - scripts/generate_dataset.ipynb | 161 ++++++++++++++++++++++++ test/test_crop_coco_dataset.py | 36 ------ 9 files changed, 231 insertions(+), 56 deletions(-) create mode 100644 scripts/generate_dataset.ipynb delete mode 100644 test/test_crop_coco_dataset.py diff --git a/.gitignore b/.gitignore index 4c7b546..a9f732d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .vscode/** datasets/ +scripts/dummy_dataset/ **wandb/ lightning_logs** **.ckpt diff --git a/README.md b/README.md index 56cb60b..4fd7214 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

Pytorch Keypoint Detection

-This repo contains a Python package for 2D keypoint detection using [Pytorch Lightning](https://pytorch-lightning.readthedocs.io/en/latest/) and [wandb](https://docs.wandb.ai/). Keypoints are trained using Gaussian Heatmaps, as in [Jakab et Al.](https://proceedings.neurips.cc/paper/2018/hash/1f36c15d6a3d18d52e8d493bc8187cb9-Abstract.html) or [Centernet](https://github.com/xingyizhou/CenterNet). +A Framework for keypoint detection using [Pytorch Lightning](https://pytorch-lightning.readthedocs.io/en/latest/) and [wandb](https://docs.wandb.ai/). Keypoints are trained with Gaussian Heatmaps, as in [Jakab et Al.](https://proceedings.neurips.cc/paper/2018/hash/1f36c15d6a3d18d52e8d493bc8187cb9-Abstract.html) or [Centernet](https://github.com/xingyizhou/CenterNet). This package is been used for research at the [AI and Robotics](https://airo.ugent.be/projects/computervision/) research group at Ghent University. You can see some applications below: The first image shows how this package is used to detect corners of cardboard boxes, in order to close the box with a robot. The second example shows how it is used to detect a varying number of flowers.
@@ -10,15 +10,16 @@ This package is been used for research at the [AI and Robotics](https://airo.uge ## Main Features +- The detector can deal with an **arbitrary number of keypoint channels**, that can contain **a varying amount of keypoints**. You can easily configure which keypoint types from the COCO dataset should be mapped onto the different channels of the keypoint detector. This flexibility allows to e.g. combine different semantic locations that have symmetries onto the same channel to overcome this ambiguity. +- We use the standard **COCO dataset format**. -- This package contains **different backbones** (Unet-like, dilated CNN, Unet-like with pretrained ConvNeXt encoder). Furthermore you can easily add new backbones or loss functions. The head of the keypoint detector is a single CNN layer. -- The package uses the often-used **COCO dataset format**. -- The detector can deal with an **arbitrary number of keypoint channels**, that can contain **a varying amount of keypoints**. You can easily configure which keypoint types from the COCO dataset should be mapped onto the different channels of the keypoint detector. -- The package contains an implementation of the Average Precision metric for keypoint detection. -- Extensive **logging to wandb is provided**: The loss for each channel is logged, together with the AP metrics for all specified treshold distances. Furthermore, the raw heatmaps, detected keypoints and ground truth heatmaps are logged at every epoch for the first batch to provide insight in the training dynamics and to verify all data processing is as desired. +- **different backbones** can be used (Unet-like, dilated CNN, Unet-like with pretrained encoders). Furthermore you can easily add new backbones or loss functions. The head of the keypoint detector is a single CNN layer. + +- The package contains an implementation of the Average Precision metric for keypoint detection. The threshold distance for classification of detections as FP or TP is based on L2 distance between the keypoints and ground truth keypoints. +- Extensive **logging to wandb is provided**: The train/val loss for each channel is logged, together with the AP metrics for all specified treshold distances and all channels. Furthermore, the raw heatmaps, detected keypoints and ground truth heatmaps are logged to provide insight in the training dynamics and to verify all data processing is as desired. - All **hyperparameters are configurable** using a python argumentparser or wandb sweeps. -note: this is the second version of the package, for the older version that used a custom dataset format, see the github releases. +note: this package is still under development and we make no commitment on backwards compatibility nor reproducibility on the main branch. If you need this, it is best to pin a single commit. TODO: add integration example. @@ -43,7 +44,9 @@ For an example, see the `test_dataset` at `test/test_dataset`. ### Labeling -If you want to label data, we provide integration with the [CVAT](https://github.com/opencv/cvat) labeling tool: You can annotate your data and export it in their custom format, which can then be converted to COCO format. Take a look [here](labeling/Readme.md) for more information on this workflow and an example. To visualize a given dataset, you can use the `keypoint_detection/utils/visualization.py` script. +If you want to label data, we use[CVAT](https://github.com/opencv/cvat) labeling tool. The flow and the code to create COCO keypoints datasets is all available in the [airo-dataset-tools](https://github.com/airo-ugent/airo-mono/tree/main) package. + +It is best to label your data with floats that represent the subpixel location of the keypoints. This allows for more precise resizing of the images later on. The keypoint detector cast them to ints before training to obtain the pixel they belong to (it does not support sub-pixel detections). ## Training @@ -57,6 +60,27 @@ A minimal sweep example is given in `test/configuration.py`. The same content s To create your own configuration: run `python train.py -h` to see all parameter options and their documentation. +## Metrics + +TO calculate AP, precision or recall, the detections need to be classified into False Positives and False negatives as for object detection or instance segmentation. + +This package simply uses a number of euclidian pixel distance thresholds. You can set the euclidian distances for which you want to calculate the metrics in the hyperparameters. + +Pixel perfect keypoints have a pixel distance of 0, so if you want a metric for pixel-perfect keypoints you should add a threshold distance of 0. + +Usually it is best to calculate the real-world deviations (in cm) that are acceptable and then determine the threshold(s) (in pixels) you are interested in. + +In general a lower threshold will result in a lower metric. The size of this gap is determined by the 'ambiguity' of your dataset and/or the accuracy of your labels. + +#TODO: add a figure to illustrate this. + + +We do not use OKS as in COCO for 2 reasons: +1. it requires bbox annotations, which are not always required for keypoint detection itself and represent additional label effort. +2. More importantly, in robotics the size of an object does not always correlate with the required precision. If a large and a small mug stand on a table, they require the same precise localisation of keypoints for a robot to grasp them even though their apparent size is different. +3. (you need to estimate label variance, though you could simply set k=1 and skip this part) + + ## Using a trained model (Inference) During training Pytorch Lightning will have saved checkpoints. See `scripts/checkpoint_inference.py` for a simple example to run inference with a checkpoint. For benchmarking the inference (or training), see `scripts/benchmark.py`. @@ -67,7 +91,20 @@ For benchmarking the inference (or training), see `scripts/benchmark.py`. ## Note on performance -- Keep in mind that the Average Precision is a very expensive operation, it can easily take as long to calculate the AP of a .1 data split as it takes to train on the remaining 90% of the data. Therefore it makes sense to use the metric sparsely. The AP will always be calculated at the final epoch, so for optimal train performance (w/o intermediate feedback), you can e.g. set the `ap_epoch_start` parameter to your max number of epochs + 1. +- Keep in mind that calculating the Average Precision is expensive operation, it can easily take as long to calculate the AP of a .1 data split as it takes to train on the remaining 90% of the data. Therefore it makes sense to use the metric sparsely, for which hyperparameters are available. The AP will always be calculated at the final epoch. + +## Note on top-down vs. bottom-up keypoint detection. +There are 2 ways to do keypoint detection when multiple instances are present in an image: +1. first do instance detection and then detect keypoints on a crop of the bbox for each instance +2. detect keypoints on the full image. + +Option 1 suffers from compounding errors (if the instance is not detected, no keypoints will be detected) and/or requires you to train (and hence label) an object detector. +Option 2 can have lower performance for the keypoints (more 'noise' in the image that can distract the detector) and if you have multiple keypoints / instance as well as multiple instances per image, you need to do keypoint association. + +This repo is somewhat agnostic to that choice. +For 1: crop your dataset upfront and train the detector on those crops, at inference: chain the object detector and the keypoint detector. +for 2: If you can do the association manually, simply do it after inference. However this repo does not offer learning the associations as in the [Part Affinity Fields]() paper. + ## Rationale: TODO diff --git a/keypoint_detection/data/coco_dataset.py b/keypoint_detection/data/coco_dataset.py index 04be705..0a355e0 100644 --- a/keypoint_detection/data/coco_dataset.py +++ b/keypoint_detection/data/coco_dataset.py @@ -1,5 +1,6 @@ import argparse import json +import math import typing from collections import defaultdict from pathlib import Path @@ -97,6 +98,18 @@ def __getitem__(self, index) -> Tuple[torch.Tensor, IMG_KEYPOINTS_TYPE]: image = image[..., :3] keypoints = self.dataset[index][1] + + # convert all keypoints to integers values. + # COCO keypoints can be floats if they specify the exact location of the keypoint (e.g. from CVAT) + # even though COCO format specifies zero-indexed integers (i.e. every keypoint in the [0,1]x [0.1] pixel box becomes (0,0) + # we convert them to ints here, as the heatmap generation will add a 0.5 offset to the keypoint location to center it in the pixel + # the distance metrics also operate on integer values. + + # so basically from here on every keypoint is an int that represents the pixel-box in which the keypoint is located. + keypoints = [ + [[math.floor(keypoint[0]), math.floor(keypoint[1])] for keypoint in channel_keypoints] + for channel_keypoints in keypoints + ] if self.transform: transformed = self.transform(image=image, keypoints=keypoints) image, keypoints = transformed["image"], transformed["keypoints"] diff --git a/keypoint_detection/data/coco_parser.py b/keypoint_detection/data/coco_parser.py index a35c03e..192809e 100644 --- a/keypoint_detection/data/coco_parser.py +++ b/keypoint_detection/data/coco_parser.py @@ -51,6 +51,8 @@ class CocoKeypointAnnotation(BaseModel): image_id: ImageID num_keypoints: Optional[int] + # COCO keypoints can be floats if they specify the exact location of the keypoint (e.g. from CVAT) + # even though COCO format specifies zero-indexed integers (i.e. every keypoint in the [0,1]x [0.1] pixel box becomes (0,0) keypoints: List[float] # TODO: add checks. diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index 748582c..78cb119 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -116,7 +116,7 @@ def __init__( # parse the gt pixel distances if isinstance(maximal_gt_keypoint_pixel_distances, str): maximal_gt_keypoint_pixel_distances = [ - float(val) for val in maximal_gt_keypoint_pixel_distances.strip().split(" ") + int(val) for val in maximal_gt_keypoint_pixel_distances.strip().split(" ") ] self.maximal_gt_keypoint_pixel_distances = maximal_gt_keypoint_pixel_distances @@ -395,7 +395,7 @@ def compute_and_log_metrics_for_channel( ap_metrics = metrics.compute() print(f"{ap_metrics=}") for maximal_distance, ap in ap_metrics.items(): - self.log(f"{training_mode}/{channel}_ap/d={maximal_distance}", ap) + self.log(f"{training_mode}/{channel}_ap/d={float(maximal_distance):.1f}", ap) mean_ap = sum(ap_metrics.values()) / len(ap_metrics.values()) diff --git a/keypoint_detection/models/metrics.py b/keypoint_detection/models/metrics.py index 52a1421..cdda141 100644 --- a/keypoint_detection/models/metrics.py +++ b/keypoint_detection/models/metrics.py @@ -43,17 +43,17 @@ class ClassifiedKeypoint(DetectedKeypoint): unsafe_hash -> dirty fix to allow for hash w/o explictly telling python the object is immutable. """ - threshold_distance: float + threshold_distance: int true_positive: bool def keypoint_classification( detected_keypoints: List[DetectedKeypoint], ground_truth_keypoints: List[Keypoint], - threshold_distance: float, + threshold_distance: int, ) -> List[ClassifiedKeypoint]: """Classifies keypoints of a **single** frame in True Positives or False Positives by searching for unused gt keypoints in prediction probability order - that are within distance d of the detected keypoint. + that are within distance d of the detected keypoint (greedy matching). Args: detected_keypoints (List[DetectedKeypoint]): The detected keypoints in the frame @@ -73,7 +73,8 @@ def keypoint_classification( matched = False for gt_keypoint in ground_truth_keypoints: distance = detected_keypoint.l2_distance(gt_keypoint) - if distance < threshold_distance: + # add small epsilon to avoid numerical errors + if distance <= threshold_distance + 1e-5: classified_keypoint = ClassifiedKeypoint( detected_keypoint.u, detected_keypoint.v, @@ -209,7 +210,7 @@ class KeypointAPMetrics(Metric): full_state_update = False - def __init__(self, keypoint_threshold_distances: List[float], dist_sync_on_step=False): + def __init__(self, keypoint_threshold_distances: List[int], dist_sync_on_step=False): super().__init__(dist_sync_on_step=dist_sync_on_step) self.ap_metrics = [KeypointAPMetric(dst, dist_sync_on_step) for dst in keypoint_threshold_distances] diff --git a/keypoint_detection/utils/heatmap.py b/keypoint_detection/utils/heatmap.py index 3ed72ef..b44c3fc 100644 --- a/keypoint_detection/utils/heatmap.py +++ b/keypoint_detection/utils/heatmap.py @@ -52,10 +52,6 @@ def generate_channel_heatmap( Torch.tensor: A Tensor with the combined heatmaps of all keypoints. """ - # cast keypoints (center) to ints to make grid align with pixel raster. - # Otherwise, the AP metric for d = 1 will not result in 1 - # if the gt_heatmaps are used as input. - assert isinstance(keypoints, torch.Tensor) if keypoints.numel() == 0: diff --git a/scripts/generate_dataset.ipynb b/scripts/generate_dataset.ipynb new file mode 100644 index 0000000..9a7b768 --- /dev/null +++ b/scripts/generate_dataset.ipynb @@ -0,0 +1,161 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Generate a COCO keypoints dataset of black images with circles on it for integration testing of the keypoint detector. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 192, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: distinctipy in /fast_storage_2/symlinked_homes/tlips/conda/.conda/envs/keypoint-detection/lib/python3.9/site-packages (1.2.2)\n", + "Requirement already satisfied: numpy in /home/tlips/.local/lib/python3.9/site-packages (from distinctipy) (1.25.2)\n" + ] + } + ], + "source": [ + "import cv2\n", + "import numpy as np \n", + "from airo_dataset_tools.data_parsers.coco import CocoKeypointAnnotation, CocoImage, CocoKeypointCategory, CocoKeypointsDataset\n", + "import pathlib\n", + "!pip install distinctipy\n", + "import distinctipy" + ] + }, + { + "cell_type": "code", + "execution_count": 193, + "metadata": {}, + "outputs": [], + "source": [ + "n_images = 500\n", + "n_categories = 2\n", + "max_category_instances_per_image = 2\n", + "\n", + "image_resolution = (128, 128)\n", + "circle_radius = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 194, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "DATA_DIR = pathlib.Path(\"./dummy_dataset\")\n", + "DATA_DIR.mkdir(exist_ok=True)\n", + "IMAGE_DIR = DATA_DIR / \"images\"\n", + "IMAGE_DIR.mkdir(exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 195, + "metadata": {}, + "outputs": [], + "source": [ + "categories = []\n", + "for category_idx in range(n_categories):\n", + " coco_category = CocoKeypointCategory(\n", + " id=category_idx,\n", + " name=f\"dummy{category_idx}\",\n", + " supercategory=f\"dummy{category_idx}\",\n", + " keypoints=[f\"dummy{category_idx}\"]\n", + " )\n", + " categories.append(coco_category)" + ] + }, + { + "cell_type": "code", + "execution_count": 196, + "metadata": {}, + "outputs": [], + "source": [ + "category_colors = distinctipy.get_colors(n_categories)\n", + "category_colors = [tuple([int(c * 255) for c in color]) for color in category_colors]" + ] + }, + { + "cell_type": "code", + "execution_count": 197, + "metadata": {}, + "outputs": [], + "source": [ + "coco_images = []\n", + "cococ_annotations = []\n", + "\n", + "coco_instances_coutner = 0\n", + "for image_idx in range(n_images):\n", + " img = np.zeros((image_resolution[1],image_resolution[0],3), dtype=np.uint8)\n", + " coco_images.append(CocoImage(id=image_idx, file_name=f\"images/img_{image_idx}.png\", height=image_resolution[1], width=image_resolution[0]))\n", + " for category_idx in range(n_categories):\n", + " n_instances = np.random.randint(0, max_category_instances_per_image+1)\n", + " for instance_idx in range(n_instances):\n", + " x = np.random.randint(2, image_resolution[0])\n", + " y = np.random.randint(2, image_resolution[1])\n", + " img = cv2.circle(img, (x, y), circle_radius, category_colors[category_idx], -1)\n", + " cococ_annotations.append(CocoKeypointAnnotation(\n", + " id=coco_instances_coutner,\n", + " image_id=image_idx,\n", + " category_id=category_idx,\n", + " # as in coco datasets: zero-index, INT keypoints.\n", + " # but add some random noise (simulating dataset with the exact pixel location instead of the zero-index int location)\n", + " # to test if the detector can deal with this\n", + " keypoints=[x + np.random.rand(1).item(), y + np.random.rand(1).item(), 1],\n", + " num_keypoints=1,\n", + " ))\n", + " coco_instances_coutner += 1\n", + "\n", + " cv2.imwrite(str(DATA_DIR / \"images\"/f\"img_{image_idx}.png\"), img)\n", + "\n", + "coco_dataset = CocoKeypointsDataset(\n", + " images=coco_images,\n", + " annotations=cococ_annotations,\n", + " categories=categories,\n", + ")\n", + "\n", + "with open(DATA_DIR / \"dummy_dataset.json\", \"w\") as f:\n", + " f.write(coco_dataset.json())\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "keypoint-detection", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test/test_crop_coco_dataset.py b/test/test_crop_coco_dataset.py deleted file mode 100644 index f1af738..0000000 --- a/test/test_crop_coco_dataset.py +++ /dev/null @@ -1,36 +0,0 @@ -import shutil -import unittest -from pathlib import Path - -import numpy as np - -from keypoint_detection.data.coco_dataset import COCOKeypointsDataset -from labeling.scripts.crop_coco_dataset import create_cropped_dataset - -from .configuration import DEFAULT_HPARAMS - - -class TestCropCocoDataset(unittest.TestCase): - def test_crop_coco_dataset(self): - annotations_filename = "coco_dataset.json" - input_json_dataset_path = Path(__file__).parents[0] / "test_dataset" / annotations_filename - output_dataset_path = create_cropped_dataset(input_json_dataset_path, 32, 32) - print(output_dataset_path) - - output_json_dataset_path = Path(output_dataset_path) / annotations_filename - - # Check whether the new coords are half of the old, because image resolution was halved. - channel_config = DEFAULT_HPARAMS["keypoint_channel_configuration"] - dataset_old = COCOKeypointsDataset(input_json_dataset_path, channel_config) - dataset_new = COCOKeypointsDataset(output_json_dataset_path, channel_config) - - for item_old, item_new in zip(dataset_old, dataset_new): - _, keypoint_channels_old = item_old - _, keypoint_channels_new = item_new - - for channel_old, channel_new in zip(keypoint_channels_old, keypoint_channels_new): - for keypoint_old, keypoint_new in zip(channel_old, channel_new): - print(keypoint_old, keypoint_new) - assert np.allclose(np.array(keypoint_old) / 2.0, np.array(keypoint_new)) - - shutil.rmtree(output_dataset_path) From 3e78e91ebe85c0598e6365367ff5cad34ecbb322 Mon Sep 17 00:00:00 2001 From: tlpss Date: Wed, 30 Aug 2023 12:05:34 +0200 Subject: [PATCH 23/45] remove deprecated labeling workflow --- labeling/Readme.md | 76 ------ labeling/__init__.py | 0 labeling/convert_cvat_to_coco.py | 232 ------------------ labeling/docs/cvat_example_setup.png | Bin 27324 -> 0 bytes labeling/docs/cvat_setup.md | 4 - labeling/example/annotations.json | 109 -------- labeling/example/annotations.xml | 76 ------ labeling/example/coco.json | 106 -------- .../example/coco_category_configuration.json | 19 -- labeling/example/images/1.jpeg | Bin 4297 -> 0 bytes labeling/example/images/2.jpeg | Bin 3034 -> 0 bytes labeling/example/images/3.jpeg | Bin 4498 -> 0 bytes labeling/example/images/4.jpeg | Bin 6624 -> 0 bytes labeling/file_loading.py | 32 --- labeling/parsers/coco_categories_parser.py | 21 -- labeling/parsers/cvat_keypoints_parser.py | 102 -------- labeling/requirements.txt | 3 - labeling/scripts/crop_coco_dataset.py | 113 --------- 18 files changed, 893 deletions(-) delete mode 100644 labeling/Readme.md delete mode 100644 labeling/__init__.py delete mode 100644 labeling/convert_cvat_to_coco.py delete mode 100644 labeling/docs/cvat_example_setup.png delete mode 100644 labeling/docs/cvat_setup.md delete mode 100644 labeling/example/annotations.json delete mode 100644 labeling/example/annotations.xml delete mode 100644 labeling/example/coco.json delete mode 100644 labeling/example/coco_category_configuration.json delete mode 100644 labeling/example/images/1.jpeg delete mode 100644 labeling/example/images/2.jpeg delete mode 100644 labeling/example/images/3.jpeg delete mode 100644 labeling/example/images/4.jpeg delete mode 100644 labeling/file_loading.py delete mode 100644 labeling/parsers/coco_categories_parser.py delete mode 100644 labeling/parsers/cvat_keypoints_parser.py delete mode 100644 labeling/requirements.txt delete mode 100644 labeling/scripts/crop_coco_dataset.py diff --git a/labeling/Readme.md b/labeling/Readme.md deleted file mode 100644 index f17ca67..0000000 --- a/labeling/Readme.md +++ /dev/null @@ -1,76 +0,0 @@ -# CVAT to COCO Keypoints - -This readme defines a workflow to label semantic keypoints on images using [CVAT](https://www.cvat.ai/) and to convert them to the [COCO keypoints format](https://cocodataset.org/#format-data). - This package contains parsers for the different dataset formats and code to convert from the CVAT Image 1.1 format to COCO format. - - - -## Labeling use case analysis -- **we want to label semantic keypoints on images**. -- There can be **multiple categories / classes of objects in the images**. Each category can have 0 - N instances in each image. (think about categories/classes as objects that you could draw a bounding box or segmentation mask for). -- Each category has a number of **semantic types** of keypoints that are of interest. E.g. arms, shoulders, head,.. for the person category. -- Each semantic type can contain multiple keypoints (e.g. a human has 2 shoulders, a cardboard box has 4 corners). Although you could label these separately (and for humans this is very natural as humans have a front/back side, unlike boxes for which there is no semantic difference between the corners), this creates a burden as you have to do this in a geometrically consistent way by e.g. always labeling most topleft corner of a box as 'corner1'. This is easily done afterwards using e.g. the quadrant of each corner and asking the labeler to do so only leads to more work and possible inaccuracies. Therefore each semantic type can have 0 - K keypoints. - -**So each image has N_i instances of each of the K categories and each instance has the M semantic types of that category, where each each type has S_i keypoints.** - -We hence need to be able to -- group the keypoints of a single instance of a category together -- and to label multiple keypoints under one semantic type and later separate them for the COCO format (which does not allow for multiple keypoints for a single type). - - -## CVAT configuration -In CVAT we create a **label** for each **semantic type** of each **category** using the naming convention **{category}.{semantic_type}**. You then label all keypoints of a single type in the image. If there are multiple instances of a single category, you group them together using the Grouping Tool in CVAT, if there is only one instance of a category, there is no need to manually group them together. - -After labeling, the annotations XML can be downloaded. - -## Converting CVAT to COCO -- clone this repo and pip install the requirements of this package. -- [set up CVAT](docs/cvat_setup.md) -- create a task and define your labels. -- label your images. -- export the annotations XML in the CVAT images format. -- create a Category configuration that specifies for each category: - - its name - - its desired ID - - its supercategory - - its semantic types and how much keypoints each type has - - This configuration is then used to create the COCO object categories and define all the keypoints for each categorie. - -- then, from this readme's folder, run `python convert_cvat_to_coco.py --cvat_xml_file example/annotations.xml --coco_categories_config_path example/coco_category_configuration.json` to create a `coco.json` annotation file. You should now have a COCO dataset annotation file, that you can use for example with - -## Example -There is an example included for 4 images containing a number of tshirts. -The desired categories configuration is as follows (see `examples/coco_category_configuration.json`) -``` -{ - "categories": [ - { - "name": "tshirt", - "supercategory": "cloth", - "id": 23, - "semantic_types": [ - { - "name": "neck", - "n_keypoints": 1 - }, - { - "name": "shoulder", - "n_keypoints": 2 - } - ] - } - ] -} -``` - -which implies we have 1 object class that has 2 semantic types: 'neck' with 1 keypoint and 'shoulder' with 2 keypoints (left/right, which is an artificial example as a tshirt has a front and back side and hence there is no ambiguity) - -To label this configuration, we create 2 labels in cvat: -![alt text](docs/cvat_example_setup.png). - - -One image contains 2 instances of the tshirt, which should hence be grouped using CVATs group_id. One image is not labeled to show this can be dealt with. Another image is partially labeled to simulate partial visibility. -You can find the resulting CVAT annotations in `example/annotations.xml`. - -You can now finally convert the CVAT annotations to COCO annotation format, which results in the `example/coco.json` file. diff --git a/labeling/__init__.py b/labeling/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/labeling/convert_cvat_to_coco.py b/labeling/convert_cvat_to_coco.py deleted file mode 100644 index 79da86e..0000000 --- a/labeling/convert_cvat_to_coco.py +++ /dev/null @@ -1,232 +0,0 @@ -from __future__ import annotations - -import json -from typing import List - -import tqdm - -from keypoint_detection.data.coco_parser import CocoImage, CocoKeypointAnnotation, CocoKeypointCategory, CocoKeypoints -from labeling.file_loading import get_dict_from_json, get_dict_from_xml -from labeling.parsers.coco_categories_parser import COCOCategoriesConfig, COCOCategoryConfig -from labeling.parsers.cvat_keypoints_parser import CVATKeypointsParser, ImageItem, Point - - -def cvat_image_to_coco( - cvat_xml_path: str, coco_category_configuration_path: str, image_folder: str = "images" -) -> dict: - """Function that converts an annotation XML in the CVAT 1.1 Image format to the COCO keypoints format. - - This function supports: - - multiple categories (box, tshirt); - - multiple semantic types for each category ("corners", "flap_corners") - - multiple keypoints for a single semantic type (a box has 4 corners) to facilitate fast labeling (no need to label each corner with a separate label, which requires geometric consistency) - - occluded or invisible keypoints for each type - - It requires the CVAT dataset to be created by using labels formatted as ., using the group_id to group multiple instances together. - if only a single instance is present, the group id is set to 1 by default so you don't have to do this yourself. - - To map from the CVAT labels to the COCO categories, you need to specify a configuration. - See the readme for more details and an example. - - This function is rather complex unfortunately, but at a high level it performs the following: - # for all categories in the config: - # for all images: - # create COCO Image - # find number of category instances in that images - # for each instance in the image: - # for all semantic types in the category: - # find all keypoints of that type for that instance in the current image - # create a COCO Annotation for the current instance of the category - - Args: - cvat_xml_path (str): _description_ - coco_category_configuration_path (str): _description_ - - Returns: - (dict): a COCO dict that can be dumped to a JSON. - """ - cvat_dict = get_dict_from_xml(cvat_xml_path) - cvat_parsed = CVATKeypointsParser(**cvat_dict) - - category_dict = get_dict_from_json(coco_category_configuration_path) - parsed_category_config = COCOCategoriesConfig(**category_dict) - - # create a COCO Dataset Model - coco_model = CocoKeypoints(images=[], annotations=[], categories=[]) - - annotation_id_counter = 1 # counter for the annotation ID - - print("starting CVAT Image -> COCO conversion") - for category in parsed_category_config.categories: - print(f"converting category {category.name}") - category_name = category.name - category_keypoint_names = get_coco_keypoint_names_from_category_config(category) - coco_model.categories.append( - CocoKeypointCategory( - id=category.id, - name=category.name, - supercategory=category.supercategory, - keypoints=category_keypoint_names, - ) - ) - - for cvat_image in tqdm.tqdm(cvat_parsed.annotations.image): - coco_image = CocoImage( - file_name=f"{image_folder}/{cvat_image.name}", - height=int(cvat_image.height), - width=int(cvat_image.width), - id=int(cvat_image.id) + 1, - ) - coco_model.images.append(coco_image) - n_image_category_instances = get_n_category_instances_in_image(cvat_image, category_name) - for instance_id in range(1, n_image_category_instances + 1): # IDs start with 1 - instance_category_keypoints = [] - for semantic_type in category.semantic_types: - keypoints = get_semantic_type_keypoints_from_instance_in_cvat_image( - cvat_image, semantic_type.name, instance_id - ) - - # pad for invisible keypoints for the given instance of the semantic type. - keypoints.extend([0.0] * (3 * semantic_type.n_keypoints - len(keypoints))) - instance_category_keypoints.extend(keypoints) - - coco_model.annotations.append( - CocoKeypointAnnotation( - category_id=category.id, - id=annotation_id_counter, - image_id=coco_image.id, - keypoints=instance_category_keypoints, - ) - ) - annotation_id_counter += 1 - return coco_model.dict(exclude_none=True) - - -### helper functions - - -def get_n_category_instances_in_image(cvat_image: ImageItem, category_name: str) -> int: - """returns the number of instances for the specified category in the CVAT ImageItem. - - This is done by finding the maximum group_id for all annotations of the image. - - Edge cases include: no Points in the image or only 1 Point in the image. - """ - if cvat_image.points is None: - return 0 - if not isinstance(cvat_image.points, list): - if get_category_from_cvat_label(cvat_image.points.label) == category_name: - return int(cvat_image.points.group_id) - else: - return 0 - max_group_id = 1 - for cvat_point in cvat_image.points: - if get_category_from_cvat_label(cvat_point.label) == category_name: - max_group_id = max(max_group_id, int(cvat_point.group_id)) - return max_group_id - - -def get_category_from_cvat_label(label: str) -> str: - """cvat labels are formatted as . - this function returns the category - """ - split = label.split(".") - assert len(split) == 2, " label was not formatted as category.semantic_type" - return label.split(".")[0] - - -def get_semantic_type_from_cvat_label(label: str) -> str: - """cvat labels are formatted as . - this function returns the semantic type - """ - split = label.split(".") - assert len(split) == 2, " label was not formatted as category.semantic_type" - return label.split(".")[1] - - -def get_coco_keypoint_names_from_category_config(config: COCOCategoryConfig) -> List[str]: - """Helper function that converts a CategoryConfiguration to a list of coco keypoints. - This function duplicates keypoints for types with n_keypoints > 1 by appending an index: - e.g. "corner", n_keypoints = 2 -> ["corner1" ,"corner2"]. - - Args: - config (dict): _description_ - - Returns: - _type_: _description_ - """ - keypoint_names = [] - for semantic_type in config.semantic_types: - if semantic_type.n_keypoints == 1: - keypoint_names.append(semantic_type.name) - else: - for i in range(semantic_type.n_keypoints): - keypoint_names.append(f"{semantic_type.name}{i+1}") - return keypoint_names - - -def get_semantic_type_keypoints_from_instance_in_cvat_image( - cvat_image: ImageItem, semantic_type: str, instance_id: int -) -> List[float]: - """Gather all keypoints of the given semantic type for this in the image. - - Args: - cvat_image (ImageItem): _description_ - semantic_type (str): _description_ - instance_id (int): _description_ - - Returns: - List: _description_ - """ - instance_id = str(instance_id) - if cvat_image.points is None: - return [0.0, 0.0, 0] - if not isinstance(cvat_image.points, list): - if ( - semantic_type == get_semantic_type_from_cvat_label(cvat_image.points.label) - and instance_id == cvat_image.points.group_id - ): - return extract_coco_keypoint_from_cvat_point(cvat_image.points, cvat_image) - else: - return [0.0, 0.0, 0] - keypoints = [] - for cvat_point in cvat_image.points: - if semantic_type == get_semantic_type_from_cvat_label(cvat_point.label) and instance_id == cvat_point.group_id: - keypoints.extend(extract_coco_keypoint_from_cvat_point(cvat_point, cvat_image)) - return keypoints - - -def extract_coco_keypoint_from_cvat_point(cvat_point: Point, cvat_image: ImageItem) -> List: - """extract keypoint in coco format (u,v,f) from cvat annotation point. - Args: - cvat_point (Point): _description_ - cvat_image (ImageItem): _description_ - - Returns: - List: [u,v,f] where u,v are the coords scaled to the image resolution and f is the coco visibility flag. - see the coco dataset format for more details. - """ - u = float(cvat_point.points.split(",")[0]) - v = float(cvat_point.points.split(",")[1]) - f = ( - 1 if cvat_point.occluded == "1" else 2 - ) # occluded = 1 means not visible, which is 1 in COCO; visible in COCO is 2 - return [u, v, f] - - -if __name__ == "__main__": - """ - example usage: - - python convert_cvat_to_coco.py --cvat_xml_file example/annotations.xml --coco_categories_config_path example/coco_category_configuration.json - """ - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--cvat_xml_file", type=str, required=True) - parser.add_argument("--coco_categories_config_path", type=str, required=True) - - args = parser.parse_args() - coco = cvat_image_to_coco(args.cvat_xml_file, args.coco_categories_config_path) - with open("coco.json", "w") as file: - json.dump(coco, file) diff --git a/labeling/docs/cvat_example_setup.png b/labeling/docs/cvat_example_setup.png deleted file mode 100644 index e83889351512eca0ad424bc6f90198736528ca0b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27324 zcmc$_bySaW7jJP0f#Mn{?gV#tcM>eH z)4soNuf5iH{y6)O@0?l5%H)xGX70J4nOm+a^rM0#?o-mI_wL=pm6if4-@Eq+^S$rz z&jZYn#VImF%+C{hDJ`da_wd^PeD5bP;qelL%eF+iUyZ^U_-vuAox3Vfiq*w8$59$}<{yqA8V8xz4;3nwn zdukEb(4lJ{cLA($!`(xg=1n6UZddpG-&STFj#tF9k1I#lcNj0{)ah%S%-#|hQ~o`< zrLwTFK+prh91?-q>y&kEI|EPuo_UvfibOVb4w1=O{QZBA60Z#VZ%vro5dUw*oxlCx zR*vi;B7cu8EkF1ATlwAg-^UO#_3-bx|Cxr~9lFD(&OOVZrB;7sHeb72)}>X?B*2L& zjyE!qr?!VT!Th*di2U|Wx&a+%*}03JFKNHl_HYcQ4C7P`xVa$|&QC77|CBL<&I9#YVL-Ug9#WkAD`b~^v@^8`_E8({cC;nzGh~GHSU!@<4xqw+=v$@DP5WS zO*b%;CRf!XO;jY~CpS2E%(PI!bi(Ux@j9!e`FmX65=tVp*wjwFCz=KQJqew_Xya=i z&AiZKb#``t^99UG*S>j_*HeC#CQD;=DC&p_Is#)Np(mb~C^s}S-Psa)8 zMyqA<{WicSf5dcA_vOfp zKg0b&=5rTR2{Sj5aW?)_ZW;#!C*VW6X;ml57ShMV!@W~Eg*ZueBj3}&;29pqKXQ#3 zR9b(*HWasSR4A#G zw~T^nQb_oD)6hXtdGC3^GUf^W|nr@#KrV0>kWVBf>n^LHA;ie48n$dSKWgB%V>NL`o zu6kNbDmX0?$l^PTJQNy#6|PJWKg{UIynjhr7R<(#cq>Tz{uKqLGe=X6xrdu|Y`c-Q zz|*v9J9x%n43QHeL3qJd z>k@gk$EF3V@~qD%stQk}drsuZDizedq>QwmQ_6wYvB&9vKsvhf;h)Z9a3!aOB9hY? z*WqF{Ggh|3PoT?B3j84u&Hcn6MB^-gc%~Pm*|?JJH~8eR?vxO!b-w$c8m2t(*kwcc zVIRUr%%wuIe}|V%_m|9Q2IG_rOwcJeulv`*DlwB^)6TSb(ZjxRGn*a!g-JPGfI)J? zaa}0nd%+oPbU4I%hc8n*&BEMo!%SY#Wg^wmWx9UCgO3R^CROYT(8-0RXd()eCf z46p#6cpem*Pn91Py75#ag}#nkIZjxPrI_e|q?&E{`2ViAU%aF;MKh;mxb!s(c?lgZ zj{P*@hW-#p0iDCW@;Yr9Vdh_O9A;UTSvlOh6bBA^=0@P3i*c5n0X z6S;?a4|nr#_Ure(oZMjY8YiI;Ic!I6mcm=*k7v^av|`|P8kRma5*r_h1tV&Owv9p6k9Sx1A&H8B}>-*LWQULU7%TL)mPpx7B&&frj!k^+y7Swut!# zpK}%gF)(Jygy-t<9w+3(;VzR4dSnJ4v78{9V#pa5u^L?d$tH>U6^0i+y770xy*?Rh z&5QcZTc@kUf;XuZwsynIS$cNz&**?U$HBNF6yiev znK#+RDkMLdqrMl%HNR|oAn16)3umTDpKotqWD3IHVo_?N%bQ4@wN~amdXeifV0x}W zyZ1=#gy!;bdZG=h63O%kWsOBxB@5uPlRlaRPzCzOX|2#tGoIy5){zNYLP6#rr@+9D zKY?qOOpTCE9{~V#nd`z#xk;$B!*+pN*f-8wy;hMnlApxjS#)&tsxLz^(Fu|hzTusr zruUSZ?<}P)k*#~A>J_e^rMYCmSx>@!sk)v6juW=+nmP!MO?rAb zMU^hH3bh3Mm_MgKhJB^glElHvPDAGlJOKi=x!vac!pa8}2;}w4_Z+aqh;E%NClqX+ z$maE0CbIHa!EzN|bB0^vzcfwa=o^i5jHYNqB% z8Oi=C?9&9U(9_U|*ApQssa_H0#&c=qnHtImKYj`-vU&|Pj4ujAO0N7|RaJ_phvcTj zs+dU+kA6Dq^T~bLIf;JQha6j48Z&LK=7*|yi67m-F3a6jlB-*!!NybEZyi;Vva=tn z@W?(j05rAmFsj4kcj|39+k(}RuRx!wknU#Q*4zJ>OtyM-{DA2fj72H` z$B@J4SDhgci1*7((k4FO^^J6iMipCVPh*xC}5QNFqRsqa1-Vq$6TV_fAL(boq+I8yJbQv z%a)f0XjnyMYB94(Xe3*~OERDp^ZPmAdVJAfaBL=xhO}V~J~^@)@KR-?F`lnfqrijr zDRG}xOih+|EnmZYEQCA7@Vo^Go7Rm*&lsAzAB5E&1RyMCY28}iZ~QSJYeUZFS4ly> zJs}Mf14&$9)i}6i8M?AN|v~5 z2MFZtG+b=f!CgURL1AG1{Ta_RwAw-oNJ>iH|6D+R`fh43L;IbEvY{)119)|VH4NVn z0!f3Dc%%qvXI0(_tz=NJ3433R@6HeL)JGvLt=vkqFK2oa>^7$)Xb~R19rgQlWSb#` zfOK@`hT!Oz#UiAY@inW}x)rlVQ=N?bap7rFW7X$LlEd6Eez1Z{Xx??t+iG(?u=9-T z_A-Jz7N4S9_>ZN93Tv|3KgLPI-|WlO;@$k+IN-_YQ_X3#QbPR1yJM{KqF*h=l+(M> zoia7!1&JIeam_moCcw;RGgH5IKN!NmAfOfVlhG9ZpR&|4$82BMjm?LE=N z%s5sUDEhYK+9EBv_7xYC#URy)fnadZtBem_c! zLX>uTQ2+H;=LoMnt)uO3vXVwle3NW23;066)UCC+x8{{jsre>6(a6w-Hq$iTd!}jQ z#BmU+WfXy^9vQNpp3oIZ!FnQZw2%Q9kaN@b5)qHJrQb_NSNX39xy0%NjV1Q7~P zWqyWCt)_Az>kYP@N|EQ{)?xLT@09Awav=I=P6}FdPC5zczx1nx6XX;mAlUfME{^jk*I?IW2 z*eu^*o$!0E(9WW{($9cMc{M$B!9YxYbDg2W2mZ{@-+wxe-_(eDvcb?#GGf@KYOT<* zm5=Hm2=*rL7;e%1pvYN1-8q@irM~w%)KrMj#Er z-hP4IN1xMtK3p;$jZK(bK52}3#}({ns7BfZ()CwWlHr!&iT)$%c3SD8EqYEd-riR_ zhj|T)EuJ!FwF9P9rmf{J2>QMd&ImTp#n&;~yPHp1_^Ucwoh3?qH!21`cG4#dfXvS? zm5x671$(cD^ZJxup$dko}B*C1buSfidT2^n+xB&=a*sT8N0T-7HEG!YEB6+jXSM_nT16))HAx-62)aR&aX`(ap9k!z% zcy4p*>ARKtS5Oz%p|uZxgGd*lwJ1YG;e06@0<+O#d=ErU*PHX->6^Yvf1^2b*#~gD zY@eHU@8Iu0PsT_@HTj?+$^S-d?#;|Yjs6Yn+!Ji*oXYSjt@<|>w3PSyzs0KlXX8A7 zL~!r__sCP!>?HA@j(o{{0UfvGO-)xYGthYC+r&>Eg|S=zc?4H`{cn)W`7>{7fWN;t zMm_6+N2lW>Bg!f&qv2)P>2rVX|ACV~8ruKRr?0Ck-))u*@ZX|m4ZB8Oy5}8Q$wBA) z4Vzexf|&_AnI0sQz5SsdS9dDeF;}S6xKNx3cizOan)4oiZnO|m|Lu~I(+(IS?#9r3 z0fvot5XurgSvC5AeYU~s^%YuezZ-QmwJ1+DzI|3!pMJ;s)ZOqa(^g`i%a-E=)MM0_ zPc(ifyk@^7W+1KPJ6ng1bwmBg)1=bY3!kwn=K$||jK z!lHgd7~3g>r4I%pdsl8s`-({S*O;Rq(#nh-2dO{u*Z&9zWacw6(?;Ai=zg`GpXqzG ztrbW4{H9>xQat%QnD50ijrQ&U+biio{pwSyq}7Q8-N;C@AVWY5tSN?sJx~lBRU{^K zYV`6<59H`+ zKK-LHU$~^rr_w5N%y^eJVRnLb4%q($e)##3fe1gE6r|7btNbkRQliErSiA%LkJqJ@ zpAI|-@Yp@BZNwZ!asr3wj??$*w61hYVm-kEO5Z&EtZGpVmqpBjXd^U{Un*-9Q3Lkl zNtxxr4B3y>cOrc*@7YaSMT%ythN42T0@_Cm+9lo9$Pl_s)Z}K|mH{7M_(E;Ine{4Y z(mJdX{rvEiUFxSY1eQSjf@HXZDZK@7{Lx>P=J_K=Rig)Q>HBu|j2S1#!-%BIetu|r z{jnsB>~ksm1mL{(?|95g4(Wewgmtv~noQ6Ey%w60T5UjbHEk>Nox@mKV;-UnKc~i- zR?ZJS1?@TDC(#g=lzustF@J8X=xzrE1J#c?Tv`;3@2171nd-jDf?1(W?9Ea-HZ~@K zZ{mw`_}^bfy~R?)0~&33(Nlh}`lP$GKA#%S#f7u02L>nmv0>s>^D~wm5m?fzBsWog zHbNj>HoTYsRy`%R5OL@r+r;<`gpXhD*j~AxJ(#If_^FqP!|f!kM0f!G({O1S^=Hcj zF>P5YBt*MbFl)c}-3)3YY&2(RwS-1x z#W*TvFD3<{yR8aAOXVjvu8n>=iT%=XSewOkSeNv-`D-g?fc=nY%Jl0w@Xgr zpu)orK$-Mvc8A8C+CjkhxVFU?ez}-$L}b0sl}s?gu8ANW;GrDVVLfU;35-AHY#HZh z{CN{E!p-Wc3U|mK?3R(_1cOO>U44(zFM2LR#UPNKaL$@^iw4@MofR||tprr;dC@mq zewV3)qd6Lj{T6v4>nF{rKqE-^&&5+;(Zl0KT3q`2k7_ptE}_FK2O5dO>Jla*^GtI? zQ;ddwpJ(m#$)iX@W|ef3a#eliHd~V4kb%baUc0fi%sFBM_AeR~c+czNKLUXbh5RTe zflqb-2~h&KH0?Hd%;TO|I;V$1o=Tn$p{oPuz;aDeiu+3UZOr272~LV%K;u5N5#&p? z?-L3-7EpEhl%9BM;$hyQBQ6x7J5BlQKKI8Zu`!wUi3vT@%p28@UqAwbL}OKmVp@WXG{uH994eWtjlrQB~SA zk(9Vqw=Dg0L66r(m1Li>0N2CV)bX`e@7Og6u{j_cqcoXaz`SrOUgy3Sf7v3MSbpQ< zHQs7KAB2Ryy%KJAy~rModzeh8=R4SR^IlO=Ltja{LDbIdxT7E^xu-Q;P2AuBaY~5z z_>^4YoYKW-tM*k~KwYGuOH*XKM&^j@L=ltZNc;3^<4*6%sT0nHlS=N0{Q_&iaEa;> z#F80G{`1La3jFDn_PX#lVW4Ss+EkD{<5FyP{$u|us_E}toB7Wwck%aoY=jb-&KFuG zw%4ZPf}&DZ<{Ia+z8Dnx3mVWFwUpe{FH3V#Jz2{TaX9*&Au>1375@P9QebtTI+vT;hhY@PkGVw*j^ovAuBD?$X* zMkcuWEV;gnPNEarq+*-FBvpwWjxZXU<-y-<&B?Xhu#afum@Qho-lBPYzfvp1q@?lUwx{3&H8?4 zFw#{|sZ2i4^WB_>58H}kXd{1GAJ0-GsgNiWn1P&5Wsbg5hehNWv6E&~-=yxh!uW$) z$NB9Yn27=LcnD2<#qf@S-S8TlIW5l;4d~eNT#U`#lxw@A z+0#RP*#Z2nXRi9)Xd_DRchtm6>D8)!2aiK47XmBjn+2TrwJskMT)bLB`f zATXq@m?*Cki!6RAylIG2;e{Jm8+>KSDeS2Dt zt4Fb*>;9iKyVleKvXo8hQ67<3nq;tNe%wjA6P-8@EH+=a#L=Re;;SZi@A4O#o1G_| zJJwO{_11|MgI}&^g`7P?jq>_m1B>>YiHs|_%O^VZcB53NH*2chEo-}6Kfq=iIw;pF9{keX=j7zz>CSy9#ikphDK)N6ru0p< zmwA+IUr7uKWqrg%Q^VJsg!)DuFvXOb-ulLns0+0>tIQS?h?^7@5fSTZLukc4IXYAj zwTrPH=`QMIZbP3O<3i+n8CC>SMJ3S|40wk;W zM(hR1v#iiujNc+-C=l1qeaY-S_My%>+`Wg7Zv|^vat-!MUlyn2IxpfBAE#!`0HgxCymJ`ZrX>hIH;M=o^P|(>NRaeY0HTn!!D7=41&3O9Y}B59ATQTNUn)gj_&&A@3$3Ev{X>XehM zyK~fCIeBS#ewj&9#gN~hLeRY`cy>Lp>DI7+Z~7vifc37e_juW38ftf$;(;ZT`>>CR z$&yV~^PyzW{K4w4Rl`{+dOGL5VzVqLQ-s?1Ok@9DsG6c&=#FP%$@H2dOiy+DTzx|O zmxaw&udjXY*NKrzlgSM?oo-#>QW;bPM%=2ZYKaGota!2+G2{;W<iAh3STzSeP4LnhXbOWd_>ytZGt{tyRlzgOHI^}NHKYt5LApiU=I zTPq|fTG8uQWzyz)qT_0QbdqzrHvI1lkdardON2pU)OXK$p8TvCN*sik;Jbh9GGbf# zmYv)2ek>CLsTIq4nt|EmA>9tuJ2@?5?hDK7LJSPvIW#}+h{@a<{CB-!O^rp%!n-S1 zj~_|D3Bf6@x9iC@r-%w{-Bh{3ke2a)ECu-ymP4a>Bc` z5J^G7M*Q4m`}3xyXg0{9esDr&?*8rj5vfo=KOk`PGUen%rea_iVfI54n1pw{Em^IS zGUDVb#y?R{mvKgt9rE+8U8Ka0^|jlVcML?jR=8|GKejU#!dxO;n;!(hU@)TxMBYH1 zYo-T&_g}HNX1v?;ed^3luFF~%*m_?*Jek%l90K3z{w({+ANXq){#+yRsX6;HMB;xA6WT=AF8al0~>G@wf>E zrQh@Xvf%@+g8c@sE^+M1)PlU~YFc(`!Ear(Kr`7cF`La4+!Y+QjB)K|T1$c* zBKg9nEF-5(Ihj+HSDW>Mze3e6G^`=b>jBts)2-?pudtz|>uU8Cvm&|uo9NXjZTL@d zPTK>1>1#bp*6GXjCq63iS)e4@s6Loj(o9LjI|dptAGp|yZdzbSg%2)kk}O_G0ED`J zuE4phX>YdZ>o7;G(uW>~pqx*AC9A|w0F%9{6GqpL=MTo?DC9Z74^0>qWyx(cTC}d4 z6;G!w>Q-}abj)Rqpkvohr?Qq_?7={%Ld9tYKHxfBVXTII1;X)lGrY*(6cI_dg|ucshBlP0w>zxXv7RnEgF;mD zKePg|G@OCI5_Nf`6}!Th8r{E+@Q>Frl=00A9#Xfg?3AlWHi4)JUh-QXy(^i#*VND@ z=X731WeE?7j(FAa%jJs7ncjyof^88Yy@yQ#cXtGT(PhG3*N%cFl5!hiQ#fknLhs=}{8b|lYRj!gIhlDL{_+hKSYD0Z zSsQLQhZd`anN@Z%rMKnC2q31-$1=w7j>rgCFm7B!l^Q|ew-fK<%pLe4{VZNj0$B3$ zW!TfGaqZONs>MwbvPaoOv4b^fpK?CP2%h`(St6!kX3m$HxAdiHw53A@4DZz(e6=FS zEen}MG=FdqB2o)58ZdFSqb)m{;1f=s=;`(4iIq2HAL&$; zE1rAa$}XLx;p#}6Z7fQUGfuw);|(w42i;){p?W1vC5j-V<}5RAwY?+8Fpyg9Znj7+ zFRw3pR zoX>e1a?AOfYgs=6h}|Ee(z7M(jKxeG%YL=_a26WCZsQXg9|m8{p6BMOnab{memFnw zto8}&qao=V3#!+y$VJxHAJOa%iajX%uHtpmUFa$6qh|5GY#2V4I;&gQ15(vCDbpDk zS37@;*=v4M+7VZ4!#BUZl}mUzfv~kJpH1CE;^3Vax_f{9^zAzOTi0`s(+h9PBuL|{ zyxgXu`UTH7r?w5nCI)LekK3VHSW$V^CtBD54Pf|E;+-O_v_=~dPfP(LmhA4(2fO;T zEVOaU5Yy*X_fBbHpEqIx3AWMM$_DR0k36SaYhn*M249;Sq4xLzjLrJ&ekuND{e9@c z4`3w5dlD>JIljd4O(0knY&v{5O}r{##dCcUAuPT#Hyz!DV|CWqr_#Zjs1a*;<-bXy zKsBwMVxTKN#v7snXZUHJoaEf4v6oEj*A=js+rK9T)kM*rZ4}olKrM_UW zzJ{>C*(z2~7XvW(82jSlDg=V=wr%*C?NG&CbGC|0fOmHMil5&*$2!pdOL(`wiq8bS zagOoPJ1T?s-)M$V-acYD0D#A3kd)ZNi-$hnJcWNNqJWM8s-tv{C*erz%P^Bb>TEhi z>BcMvhcDb7Gw$athpx5*PMn6JhTh7L#K0@q`NA_P2JscSzdHy+nDZ3X|ABN1$#HDB z>kmhXAYYw!xmsM8CBuY2-%FYjrQ_ud)UZt$^$P!Z(HC?AC~Fm1bQf)k`)#hZ!*5{Bql z+>-q~4FgaaOYHEbkM5^8Ek{%F$NT7e@CUiEUpT`n8>%V4I`=co`{B51Pfsf{de$@C z7sE=-N>6riwf&-c{+($owOeRsxM$$f;BYbTv&EN>S>MyR%dKRw-@LkWkG=YIoKLkO z;1e`5tfGoPC#Cpa(xtA6R+V_7POqE!BPlDxI2lYP$k6)(<`Hft06VGn?&TG2RI`il zbatH0HO`5fx1#^DQtZc0AII<41hH~;;&rn!*Ek;}f&$q+fk1*z5u~~7d!+-v6D_u{ z{KBHN)bX;Ih)PtEQSNzhw&-SCdDInzaC5Nkq@rX)s?tPhEVo%fi~#2hf7? z=D(wUMYk05`XGjAv8W`duE<$Xb}3-gDEJf-*jn}!ZJ~E<_F!5p^rtVFhykC_*Q|Vs zFwPE(y(j}~sUlHTyoC^PY{5|Ggn`mFYuwrQXexn$@bq*B03hZa zPgBWG+{o$+tvUJgg=T^zDk1?osTs$TlDwkw@;#|jG=A^R;H(Zs^@$x`LJb9MH`Y$p zDnC-YLFcQ@PnK=X;Vj+x2+~5;vj*wP8W?u!U_n1G?qNmwLjc(JLqh6sOfD2 zrF{QB6<$>bNMD?N$k(G^Eu=!l>*jg(HcfapMzMN}*2!#s0rbr+4VwKV?(HnPbmcu) zePiAeOsw~X?%3vSvv#6qEq+N#N|FSFbywqSr%o~P`=#s3eE*4*L!QU6x4;pHz1f}nZ;0E3=V-Ohy)r5{9OVN>je4;wg7PVGcIVx?^4rHFF-Yc z9E^Xj6MK@+ih)ZKop5m@6N7_;O%K1B=Nf+!)A}qB6#WEcM@6x=KHl7nkGf%$-@g3j zEX?b+QAnxDux;NMHU3ZsIN2fKNVenXQBSf5vQzF9=Se2FU9 zd@P}!d2PW6QXuAuC?gARf)Yp>iI^WT+`D%Mm?$W%{bYtJv9nTi5Fmf(+!;<#=!Rdh z`cHOVKMu6Cm|dNeCzIGxLOxl5_so@2uRgU++AZ?@@WG^((KKUNMD))0cjNiZNzjh< zokO87tG{Z#l}=5zaH;jfNyG!&;hzL@Fb@@AJm8AmKD`!QM-vyF+-NNp!Y!b20HS%v=6-~3# zg&4qz!nDg*r!ts~o-Ub=L%F{Yr-!TV$Jqyx^2-WU3*9ob69=YUq2@QYzr=~4|3njn zNHY36fx3nC&%-}jNP0xPWB;vfU`Fr8{1LyDduynTTtS1#F&)+D5Olv4huCaTsh~li z+dHqC=k*8o-8)%F@vm)6e(dCNth~Xl_$C;4VAj={? z(jDxmuo&Hp+F9lwv&0V`guJZmj5GVb9MHID)_d-uji-1f_ zHf^gvdDO*M?8?BrfHT8sc&TJKWR-6yziLo*erx~G#^`DvA(!1{L|9HWrmyIoPep z(21aKY!Y5WJa#U>>lY1Bv=M`86hdofDcJ7c`yHoOFr?wsM%NRVB=!I#qg<(I&?527 zqGXIHbCnK6LtMK9-O+Ie-WMZ3m{tkuN)cFHZM~T)Dw4?01P`qzO#D=GT&`}rlzepj zS=1GMCu7CN@mb{1fL)ndBo=ym$iDHDeIcmmjRA25M};!5gdryf8T-z&BatHR?Adc1 z@ob_uGGR}jcC2Ovsv4-h3F>#4C!c4TLJg18Vz!e*q|se4+x9GAPX{ z(&JubkK^rt?2qiHkG`Vot!L0r?~iO9#jWSdHp92px!B(kyrt;G;Qm5_9&wX8FV;v= zkyDHfFRc@DbFMU2udEGo={McJZwnDX=sH(cBXZ@zV|GhNkmUh-=DsE_ zP`S;PRvsKuagKo2cGl5X*n#3$x-n}P%g08gbq_pnpT$Q$Y21OkHU21wa;;WizQ#-F zc($*p<>;x>6Cm@(hRsa_u@b2t=|1GY<<=d*i`cP zH{wZNA*NMK!T-NV!4F}Dv{_hKZ0~CSMrg~*;)JwqcRb+u%JZikXKI2_OH0eN|1-J0 z)A8WnEcFkZukT(zU!o{lQX1;{myq&5zznHvp!7<+WTR(D;Ao!rIbhu@wm>=c+P=MQ z+@6qq>#+&QRjxCysY5aSVM$kSlfL(rMWVlkN zd|H+BYR!SI(H;gwkq_eKRd~gREc5W2_AL&DSaSNoR6FGiN@;0-QK~lN}18b|E9Tf zTfcOB7Q+CLk5K7s7vVIs*jQ$6=p5hE6;>})Rqz&QF1b#5$v3FXlu&VdGH6HN)d9cg zh*bNN`NHHWR5Cj=R8kXf1x{z_w0cW3Zq~nq-Qb2N9Trd{rxr8)d{Fq@o>%@BsL?W? zLt&JJFXExYl%2+%>-Iz|X>$yI)Qjn;ec#RL$dQ_FsZ*aQO*RU(N-(Za%K|M{qiq<`<(TEGy zh0@U@ZSE%eFL2)>1pjpNi|9~NItDmJ&y+^=9CR8)D~+a)jQ3W0?+bHQ$cMgpN`BWL zYnH9!$2fgrz?E5cuG(JbuZSA^_--U~@`hug=6vcJ(b&j#7oL@u6Y9&S`f#wjj>^`X zx;HxFlH2TkF}F#}qbEAxxXf9fl`WL~>C}9t?`&@j%SBhU+4^1?D&spU)9#c00==(|k+7GLVKnyg{k^3*a_TD>#?NLVDU^(sdx0frU`%?ou+_M7yV zc7ElaS!+GGLbRE7cUeD`<*wuEKWwx_@u?MpDl>`7Rg{#q68YQoiqqs2JYM}LS z%I0l-gF}0c#@(f==DZ|ZW6a{z_Gq8eYv)K-^tFoBehJD|xb9pdmeT5^V_*X7&rII) z)G~}4p;H;BkFnh^+Gomdp*HM!cwp6S}1<3k0=BP39EBGt#~`&Dpt!oOYHa`Bm- zA!IO&rkG0i>$gs0_$(`v3XzCg1#EwIL1)K zz+_DW#4byVQ85#}7;ad!^>m#mV^?<@n_5e)UGfDvA}n7&#feBcTPdFN&}r7YTOg4> zbTNo)*B3GR+gW>@tMaOl(7VxkjSN-$ro}2Au9~n+e!KG>4*~*t&vPoj6rzQz{V4h5 z5K@0S;f8~%LSWX`Vu|Lr=WCP3W61$|gED*OF&_5Y&66T=+#{vSQ_CZy&Dfhwmy^kS z$sUkyyOqq_E4i#drY6>+C#j+?G$92OQ(HMdwCrFyY22|o$NR%6w=r@0n%9$sU-hD1 zU-7ZYQGAfcI)pKr?yEr<(VZ5zU68Jy}jze4Q|j& zuo|&(m#Jchj-bO~&V3^mMO#uH)+2;{70h0!j?FSh~-4IJeG55c#-7(bL^@#Z8Y_6TdY+V7?% zEI)4j7T(O~wvo(+n#ppZzWm$H8LO?qUi|W@&iI|Xfrx8R*dobwGr-$p$Gh%Y5?R?o zqYV2FQ?`c?!JDBNI^Oc5!S|!NAt0v$|EoxxgUM^JKU~#&7g{^1Wg^R5B)Q72op~d+ z+IH46#YvemVHlR0VhT#8x;nYLP^&O3t)`>hw~kxT^D}aq6Eh zBBZ<0!}Uys&Fo*MUHVb`Nb&0awFR$nGcOyXsGuOV$@1cf++a25v&_;oK1Nt>vC$IPOG18Do? zxnDcUpv~_B$IMToY|_d43RI|q_=6xo_OAkqq0eZZw7gya4S~7vv-?jn8@U6 z&H9gmpG%AkDS5G!&WW*m&sSl>vK_?=7-1rdmmDQIjfP0g9$)WcF^_ybd&u&0hgX@* zuJbAunVVCcY+Af307^PV--Bose=lLIf5GSX+$1w2_8l-NTfgn;3cefk?^=QVaLs_0 zrBc=4syj+R>Dd71OU0$Ev`>2l{@e8#OEPi ztID`6jjF>)iem1XlxO+V*Le$z)eobD(V04-5a-E45v#2d?);ulhHdmcLcuH?BKa@> zk|geZX~@Pm!jEC`nI;s#nQUaE^6L{nTD?1ryGbTJr;ap5omkw(v&~krzTFFR(VP(Z z-6_46S--MMdrYzHzl^`ZeRDX0oO7}e0avo(rLY13|L%If^@cEJp21Tvh z8y#BXtB_)-m830)s3SAH{|r+IW``N3-$|_5>bCVYjz})o294_7=D@S61>=!Tlg6?- z-@UX~pTS^n15vfag6LH1^nk!p*jn49hK;h7Re#=_omi^mrJn)wndYFrDP3!JwM0`m z{5{vj@Yu^>V+*Gh(sF&hWT!dfk|AUG*;pkHY2wTAGu4))S5Ty-JkR6vW{sw%im$J{ zAMza5ac4e!#q5?ydY0>9b(=BY7XKInQ0~OS0KWImmmlH%M}`#R`RQ{I3~?^hWV?Zf zY^F7WVZU^D+HFBgz{}|&Zla>*NlT{kd`12rg{9K0a(>?y2HzbKq*EnJK~ADb^$ku> zRCZIlLApF<=g{cmP-`#|C9$<;gZ<2L@a@Q&iB)39HGz`-j2G3f!^XH{=U(NIOvl#4 zyDMv9G4PYV3AD#}uV25c#THeu_gVCvb|Mqdvevi5i)BA|y>If?Q~UJO`l>}?d8};! z0<(jz`p3BbU1AS{eK)OWMqeZv`HAzH<%-MGbIl#6`*GG)Khz6*vW z;KTnlpa%8#uDROQ*4Bbyqk43~p^letaFgfCRX#=XUl{X`n%BrV8;p=7j!K(JF_%~S zbglgt7620^dJ!EV(y~oOC40Cl^S80zieLl8+F$7WJ^PUVuc_{TyYT-G0=>Ghj@Gkw zOF~@}F3(3nv~_f}5?$(aee?=5h1l5G@IqF1cQH`rKMv~`p-ieo+oLE)vD zNWuQ;UXoW0+kM@ZWqaLP`=9@z*1HYOAFNFr58a)v?eEVIkE}b^ee>{rTC2-C^B2@C zOxF272MTwy>&lp!nQ1EdyHVb|8Jlk_%df6>Z4G>K*nD?8V6!oXz7D;f1xG98=9OM- zdCjI5F2jE(o+>*APT75ljxJ?)?^n5Fu-o4y{WI))XS}us8cQ$gxDb^VqppVw^5Et> z^r4-pl|*9G^89u{=v44wHF_Vu_|~YLPxzH{NRejid{MDUF=R+6A-82tND=W;SKsGa#slCE_9I@T(oT+jOz^s&jJI#|o%l_{i__=w_ z$to=bWp{dcGQJfst@W|yQNuewFBBz-S{2IS20P%{q>}A|DM3){Z`pz;IdmqW7Rq6X zg%;aJB(jll@rfR9qm?H(a>Y`TQzqjqg0&1SzLP?p7Dk3<6$J2oH2i4%C!3m`<{QFNl&h2r>r+f4-WeU5UKm`cD>GOW@Dlgknw>fb<=G5V@Ez|^;Ij{x z>K{EoX1u3H)d^13WPEQj`7yZQ1UBp;^P?;-^ZS&d1g7yBw|YN7%0D$NP+R>^)RZC7 zYCHDcc-$=uXGinF-jD;vpX9JLtnr#p-`gkGb>277{Kg7g-))|@`eR_w78;f1>uxam z_;R`AWp+O|!;gfJ>zyHmK+ur>(9lS&3;!;=QMwX-rX9Smz5+cxAz<)`8x47n$S)VO zJV^KO+S;7!*Zpyxqe4sRtC$KbSNQ`rpl7~y%y2cF5@~) zUCrcYQ|&~u$&K-gilB;vqudH;8@cPsLAd9(|f(bRhp>C1q38CX+gSD zRhoc+5JE4~ODIx86;Lc7p;rMzk={ZHp$C)>fgl}12|d)%JA{qi`R>f_fBW0tUpspy zGw&&LPELQH^ZA_Tc}I#FB6(Qh^ryRVAi5W4mS=9oDm9PpOqcZ&c7x_~BX&wlM(BbP z($oIoh?CE^$44LFVg-UuHbWihEDUAxFc_1Z@B=a`HjB_p=Bb+w?~SjsANV?=`X*zG zSbWQUoisCsMiZe1Fo&~Ga`#w+?vHRqQ&P-a4vZ>@TGUzG^3J1_UoE?IDRzHi806tm z?>6?G+4wJzax-)I!XQi|1J+gX6+nM7@{$Vxx*e{>)n{nhjsu$W--O5zAWhYdyK$}9 zEPSVu7h`M`%QqiM;^oL#x*U3ijqPuo7Phr_g;k6$d zC6J`i#!pW_@MyyD^LW_uZpb~zG*W;Ny-!a+;n7#4d9sMPdQcBM_3q8C2Cz2C#7_ks zVjuZQ=q=m_NYXOX$0pEa9~weaSoVSnRC0#GZG)>Z10PJk0| z?@OG0Xm#{0`uM_^3uiAD^IOqMO5(&dJRu@&-?Jr)I^O67&#KE^dY4xLn+Dx#dFdx6 zp!0x!ss#G$*E@Jef>%(iGah9>q7AOKN~W+67)gv7etzy5nZpDV>R0C4+;#Qplh996 zmLFRxs80IU_!WY-dGY8=ZqsGOEJnd6xY8CKNx2X?&jsMG5Gefql>6fe0C2SfeH?_p z^oHz1CNnz&=%_LdEUyhHF3lP`##fV4_;By!G_2z6+Ui?5Nk(>fM@Pwx3Cug>!fg^3 zb~zQx6K#<7tBF(awHF&|FJ9b2Cb$kWfV7dub`*XIUU;ly~)FBUuP z1tT_cz#1)-a;nCFcW#-@(+P_o@Se2Ce>IH>{#=pRs*&p^o~=tGx)d(-!aVNv?%nMY+!Y{(Q<5_`n8k4Y(hW+ru*?zURY5iM@l zyR3w6@xPO zMm1H9DNn~byeZsQ+hP04Oq0$!}ryoJl-+8&M~3hSZSR_7E9* zcMlznQ6ji_UYA54V(h)EYk#pj(*`!Nx~uX5`@E8iHA5=&G_^WnRZ76l$EueSZ5|P& zur;Hrl+%RNe2Sx^vc5IcPT17hgw;fDiiz0y^d$9D(wckcx#N8(UMsMZhgz=$`@*AP zKxR)b@$?;|>LJOpM-_4^bnp1sWNRzqk$}fI^8l3f;dVP4o=wCq+(AkF9oI@3|OUAF&U36-#?Y&MPmvEBtI$1jsnBjjS_G&`v^Tj7&|AV=@kCIqT=|SFP?N=Whw~zZOK+^=;X^xWLoW0@CpNQC4Ix94-WK zLe*DKhTL*O;RgB0e_87}TxQXnS5!Y;kzY_yKw&~R{I|?6*MM{VCA1+5CYsg7els6$ zTmGd)9^CtFBJ2M!I5F&f6(LM6SWIllS#r!K~#1O6k zT;GqgE?QN|(#1OY;j}|i!3)EzMtl%0?&Uk?MLQ~UlCP^P=pcg?@i1){slTqgv|f9% z(cRlS?1b|6_C`&|$mp4yoBR9wFInW(eeww@+SmKoC5r-da;@)=z~pSwrG@gh2U9_x z$=uS*IO&Joea@15f#_T0%wwz_|EULHZ#Zeq+)q^7OqK4>w=Ki({L&L}p$shT{Z~w>lbTd$aa4}YO!(co!`Dct zd~yb;Je;*qxmeN=j~}=&g`RUN7Q5^d^bZ z*xsF?kV%zuuZAP3;-&N=z5~5m2UWfei3)7PF97v3#ExT$xX0SyJyR*l6r7nqsxhcL z(@r!GN$u*hB$DTO;GKtHDE+$i*>x}YQkE10>pu=C4mjxf8**z~xXfJ2s9WyPbq^Qy zbJ+8N&s5*2pZO#`%}8omNwX>ZvMKndC}l$f%v9uhipBA8;YI+~>gJo?;qDi*-cw_)7BI@>-3({4*- z^P5Y5HRi8X)0jv;%|-yMqK()ax^If-G+PEbe-+`%;knH5l~GbwUiGT4#OGvx`=*q$ z{~X^G=tG=hVqk!3=ttX-m>Z^n`+I6x?*M?ABTYK|(RdZuol|DG`--2&h==wxg*~t< zg=&e;*hFIh)oeJHH`t5xiaxHze8GAgPoa2AmE{7L{C>(US!_a^zhV-*p%N@#MQk%f zY?EaiY;D1&rcd9Xk>K2=+!Xtr&yU#i^Q_VSxu&v#eQFT$I6eq;z5d+xqoh+XP=PSr ztUvt*`NtJ#2*9fU-S~KKW~{JWHvUw%SKs>i7N)KPyZDZZa@2@-{o}Sn8alSW2;LrE>_j?#WIG}KE-@Y}G2vFw+bVLlL0ElEe z0oVNkcI4`4gxJMA1NMitlR8+tkG8L~$-AGv0(;?u2Kfa9f}Kmw#H??y z?!P!*+@9^QwUAsTpwC-84r;mg&H}<$4L3IE?1GyyF1kOb^oy0FMeS0@dbNam{tlx|3Kn58C{A@!6D)Z~OU;kh4+ZI*f}7voMxBnQh- zRb%Qp6X#O%uGGNQBe*ZJ{L{;S>~D6sUj5gt23jqGTE} zn%CB=eQCeTL-q#AJ_sf0eD`p5u}_~U4aJ#EhIchqbCM1SaT?1Z`OH2m8*A1O7tigM z!ll(oB$fJIVJLJju+mb-1d`U)3?l0;KS-MIoHC(oIMI?m))XKFnD4xCUO%~NW@-X_ z)MTrKRK<*@K(NArGO8>MPc>ywgDW?}Z}^Y`7bih7uyB}C8?MJGvl@Y+>IEuhGG63CaYMD1;jVz!3zNs|_CTOs+* z9R_``(jIc3m=G$=(cU(!MGjfmhTu}>^%2O$M9zCgtqWv2Ifaaazc$cm!KE*U)30*Ms!`vl-&ww~8Mf8OflMkh+trHUP_vj= zi{Yf=oQVR{T{yQc)4)Rg+B8$jH+I+VgFL>@xGheBAq3ikuwVs%e{)!vW&6sQf!az$ zB1_ZRBk|zXaXPD_N)j&zN0#GIUJ^k^M<+?BMesdE(vQFBumAn`!+)ryd@YMQxNUqD zqHFoCnv)!$Qykf01*jm>Q5`JIF?heA##-)EP_^Ynj~ZOmu9=Bxi>XP{(Zv@6{VCJT z1!P|}b8J6Fcf`<1ado(TnF31e_H`W>C&TSYZpLl?2ebW>&ZNVIm=-qx4+3}wA1)7S zS-FEiZ%^BCACm1wWKgh?Q@a4nsV={01s+)` zqUsSB1>wXy&7 z5b|^lGYVVd!m**&_rPle6rD-9k%YA^uYGD| z>`y30I#<}I{WV%fc(_Nth!*Wa9RI#9a-^TMARna(Orf%4Bax0u6yI0D;Vv7>T;tP6 z?nk2nl>gSc+NCPt4?*bm`t#!PDjfv zFGi+`;h+J}Ip^_0sV}IqAgZtZ)&3o zpd8kpDD7P$)N1)(F;i2-10RfA5Qq%o~S)Qzo7`q_>KC8Tr_)&)akFK8>><*g6$u!ue4JN_yY)VP6&ugvV+E4)Coq0qe3Cig*@Ar*NY~ zPc`f=%T4)l3r?q_+4QGuow^`_b4BEHp(DMKJ701O48y~7zY7B9*X}E~S8i3=m?vx< zRL+$NR1nevJsyFgz9vtvek%=*EV80~zUJ4B6|k^>+{~6(CD<*cF@N20UT{=@e(nLC zcx-OKVvY`NMtm*jt<2l>*IhZ(c)3ZnHl_(%bkFN`QcY(p*3G4uHiR{fp0Azs$WKSw zvBCBjY*@y2VOtPo%hnl;)@-hzNzVX`%d2{ z4T6t7yJkZ|A<|RGK-=AYN3^y2cCzSR@5$1dU~yJGDRv5(zekey+Sw0TXio1HV0x{A z={qKR%@;P7uw#i86_u_f2w{M>uk1}vyCd_3op?VZ4Qi8-(F5txvh#jc?)v&u*GFlM zW4`BYxt^CYCjqNFY-}!L@R>x9xkYjzsQlaPC0n-LI-#EZ&e$opS}%klLauvLMJ&3# zE+qH`?hZY@kICramG5TxL~V~rl!zp9x(h)TCrEN3X6E73aT5x?W)T#MNeDBRqd2Qv z{J@Yd4Q)`6d2+BGxc;s3Le`9k%shJhIN>+obzQ>Z@j&%r3L>#A^vCO#1Y31Emx$ne zjY%JM^M@%HqDAF=sapAyb)NaQhY7Zt&+8aC0Uo=H-R0%w>tkR|=4N}|zF_aKmOjqC zaVsmypP0YtYptcmrlwjy+V|*m6HXRYdufOA*7DrngCyN{t96pk3T~eJkp`gL{~VA5 zdHwo%G|4hKplAOqcm~_+cvrsIzB{&+udpNGn7|eRk zRjjyogISk0u1$ENT|O4a`U&!@KVsnihC^QP(ia{lVNU7;|0=x8B#knvHn081mEc#2*bV*!dgtE0WNLGk$y_e3uCpL?gN~5sz+%J$EEip!|p5()2#)&owqn*6g8w^Aj*$rUrp`GZbXJY z*X{O1>ixt9*REO|8WRiG_#R@WHV0e4#u9P%xGw9Xqz{oM81)-4E>SJ@j~_dl$l%3B$=2mkPPeppM_*lZPdILw z%1(Ejq*IEgYfG$^&mJd{hovaUA1tX+URjyY*dYL@+*bO^v1PiNRl?s!M?PKvbR&DO zUU^a|E#KXjaHnD7pK~k@g*OX&BRXzv(AzNt07}DQPs^HRaaK*TF9xPel8eWZNy2Uj z=i`=`1e@&}t9Wp}{^vWXFKjWXFOmru7LsgC*3fje_s-#{O-^1hG+i3g+S+ag`sld* z?%g{Nzp3w5!v#7^)l_2tP}ewkKD0jHNdD9Z6l3Z)BDf0hfSP&~358m630Zs>kJ#9V zm&dTWQsclST~p}0#U45%{|ga&cz!3?t-h}1D2-i7& zNP&&T>lk@G=}d4{78;k7I;S&8ZDvhJmn_Qp80?rIT2lk#;9K1le$U6Z`WDx)gqHFz zb*jlP>Mo1*+D*tN*k1fWmamDP@pd$>Cu!;3^ehnfs>BFboVyOzf}d&gF`QGghig#2 zRK4S4c(|=v+?xjN^uxps>=**1WXif|ifA0DlTIO|27nuIm^uxqm)$}j$@{YNM4vLn zY;?y0W~fH^am9gq^r*fSG#;9yYh-lfGBtp$H-9sl<<}WB6AOJX-dFRD*`x2o=o%xM zmBlEi#J<$Yb<16J%Zf4ziQamb>!Wr*N{Ovr0e`$opX5QNc}kD^B$& z{~0rHyQ^pzf5GL@%nnr^a0;P4&CYl+Fvx{@+Zvd4G5JqY&`s7F1#olVM3wNoK(`!Q zx-{Xc2-O((di=uOCKjf6zTAP%D#%>J-u!;({tgv@k+?^@xhcGsNqT=C*YxFKK>SQh zI&;k{FTCYIVo9`c?ue5;_G_J8Uuq>jUqe}0xsQymEcd32)XIaiRwZkL7iznjyCp77hiFFv@LU-bso*8%v^q@QNt)12fm^H z>l*>tt_&!UI4$$zq&qeER|%Ov-Z#d}XVgFBkPtIt7vmIFc}A%~1WFZn2dq(;_9lPQ znqLL*L+E2kE;;Y@O<8lyKzxEl%TP4==Vx(tmqRC}kG-^LZF>fDSTM3a zD%FK;?ax4p(2HeSHhuvJ83lQdGHYaI@f&wChNRkliC_@&DfEYHZkVP!9KQ|w0)r8K zoTM`yUxH*9;b;>o%IJvmo{n-UyN%81@<0|%B#$>x(Tk42$jD=tAKta2LHLu|H{h|P z)9P@Q+XCgK-~8wvrN5LjUVo@}XvX^GkXCM|wb-Hj!1g=xn}t}NY95J0Z?YOEzDy>I zt7L$NQuwQ}^SHlaN~rVID-Xc=ymcw_!8ix%d+es(@eYZ>(XREjY4g;I6lZ8@i2vw^ ziMjqh??WoqZM~O`w#Esr+o4G6+nk&f%9H-r6!3C}pdBJAmAG&zw>aI!q-X#G`p(YI zX_sh8*o8SLDGdpU9~2)Zh(k?4roH;w=A3t_-d3jmGLo*s4hUknTiGcFZHV+Y|@>2}~$2n&7 zNPLiM%6nje#mg8wYsdB(7q7yMrYWD`Pn&@p3-U~iUL$`;;4UNU7UaKHZy%W7_86HV zWyQ-gulkIjJm}z$9up_7-(J2Z6eMJG+Nc&QcDLz{^pC|2Mp@s%MP6}T9&P=fT02y% z_bT~Bt&}=)4m{7I!%`vXOefy>2_vWJ_vEtJ8BI&iWJc_?!PD5=gXRL+GTjN9-xN3@ zhZ=4cj<1AM7#A^0-Pnf8;-?2|Rb{+%67kckacLHoC)@#xDU!QRF=1UzL$^GiC~+M$ zp1YNOI1N`3>Leu<;Fsn{Bub7Id1U$Jhz*BmRgkE_RBn9~tY3P#;@wztqrc`c1l*e< zS3O!}Aj^(KB7DT9tVuggGZ)p)aOr?CEKrC%_k@7J^x*SkLaMhvMyARz{D>*KOb%Tf zrykTdEK@YB2sOK>x5im#Pq;VjFyXk_iR_LXPTjsM*H5q*hX?A}6ZDB~kxT*8YmcKH zMt`B%9TDgztTARML+ohMXe=O4E%UuPJSlC5EG&bsL-`oB^s6+;st#bAgPMv#w3PsA zD_YL1-m5MPREb2ck2Q~`>o91jsqO6SkfYWL$Y9uq4yJPF0H=sR zr!~AL%d#xzGR-S~Kn$96b2T&8zM71I7?@EAX?wm;wj$a~STg>T10j6Nv>QY&p^D8? zxpq3sK%NsU>G~4_0gQSJf(h0}y&%67&y&Y6>Si-BMySWZ>U8TgTRNp)a zJ;!%t_OMvx^?0?5Ng5^2VlP)(4wo%%Tq-FjxPiq{Qja~;9tn9oS!R*G4;>JeV#5L&}%l|M19{Lu| z)UF@NWPv@)S*pw5u44W-uiVKa69co$%Sq|!u99hG7W;@C-NLD<9=t&LJ;WS5lV1*; zm49IaH}`lUfdiOEBaPD!i)RWgHKh@c6Tg6b3++rkOtwyR$d{;f{^DUWc}CuiFD(9Y zdXuFEleM3q<}byrJLykGey3SkoFD35Le%z)UbzB&HvMH$e^Rq{2u$V&HXX0?tE!S4 zHykeg5NB;`YikXn#QE_qFq3(wSyz2ZmEV>aXYKy|p2>fW*!jO9h5mnD^5Fcx(Ct&3 zQF5cBqjty>$>v|Q;#=u9XGs&2F&n+Q>59&dj=b{n?f<0Yl$ZY`$503~l9NV9B^4Ad z|C7X}pnxA8t^S>)nyWi8KEAlTJhOcCi-~;6OZMZ&#|!j|m`jj%tHsM}YJ>#^xqjbB zf50mxRYNX!W~RxmxD*bb5@(J~OayCbMRnU)h5k}Qn+4iSrfKqk$We)ji7MHJJEss2 z_)nUOni{FJzLS%ay1F{D%!V5Ieb*m_CEvd5=;-+IBg>9Zomhk#=LGO47ImnWk&C*L zdC%40m-^KI4p*B#@(Bk0P1NN#CJNJy&FZ%r*d?Wa%Qspl50jHHFj(U_V>vuSev zZhC?8wQJW@qw^$QyO$Qc9l>|Nr1mIyhqTY(&>3e*{0r`1HGb(ruKhb01hgr~z)(B! wFPrJ}=wI47^EH_ZCA%A{zisILw*-$ag-5siDO6WkewD1Itfho__A2Cm0QG|NK>z>% diff --git a/labeling/docs/cvat_setup.md b/labeling/docs/cvat_setup.md deleted file mode 100644 index 3c680d7..0000000 --- a/labeling/docs/cvat_setup.md +++ /dev/null @@ -1,4 +0,0 @@ -CVAT provides a docker compose file to set up everyting on a private machine, see instructions [here](https://cvat-ai.github.io/cvat/docs/administration/basics/installation/#ubuntu-1804-x86_64amd64). At the time of writing however, you need to change the container tag to `dev`, cf [this issue](https://github.com/opencv/cvat/issues/4816). - - - You can also do this setup on a remote machine, in which case you can either make the client reachable over the web or forward the tcp conncetion to your local machine using ssh: `ssh -L :: @` diff --git a/labeling/example/annotations.json b/labeling/example/annotations.json deleted file mode 100644 index 1c7d36f..0000000 --- a/labeling/example/annotations.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "info": null, - "licenses": null, - "images": [ - { - "license": null, - "file_name": "1.jpeg", - "height": 256, - "width": 256, - "id": 1 - }, - { - "license": null, - "file_name": "2.jpeg", - "height": 256, - "width": 256, - "id": 2 - }, - { - "license": null, - "file_name": "3.jpeg", - "height": 256, - "width": 256, - "id": 3 - }, - { - "license": null, - "file_name": "4.jpeg", - "height": 256, - "width": 256, - "id": 4 - } - ], - "categories": [ - { - "supercategory": "onion", - "id": 0, - "name": "onion", - "keypoints": [ - "head", - "tail" - ], - "skeleton": [ - [ - 1, - 0 - ] - ] - } - ], - "annotations": [ - { - "category_id": 0, - "id": 1, - "image_id": 1, - "num_keypoints": null, - "keypoints": [ - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0 - ] - }, - { - "category_id": 0, - "id": 2, - "image_id": 2, - "num_keypoints": null, - "keypoints": [ - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0 - ] - }, - { - "category_id": 0, - "id": 3, - "image_id": 3, - "num_keypoints": null, - "keypoints": [ - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0 - ] - }, - { - "category_id": 0, - "id": 4, - "image_id": 4, - "num_keypoints": null, - "keypoints": [ - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0 - ] - } - ] -} diff --git a/labeling/example/annotations.xml b/labeling/example/annotations.xml deleted file mode 100644 index 82e17dd..0000000 --- a/labeling/example/annotations.xml +++ /dev/null @@ -1,76 +0,0 @@ - - 1.1 - - - 3 - example-keypoints-task - 4 - annotation - 0 - - 2022-08-24 11:46:30.553731+00:00 - 2022-08-24 12:51:45.689497+00:00 - Train - 0 - 3 - - - - 3 - 0 - 3 - http://localhost:8080/?id=3 - - - - tlips - thomas.lips@ugent.be - - - - - - - - 2022-08-24 12:51:55.830700+00:00 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/labeling/example/coco.json b/labeling/example/coco.json deleted file mode 100644 index a045df1..0000000 --- a/labeling/example/coco.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "images": [ - { - "file_name": "images/1.jpeg", - "height": 256, - "width": 256, - "id": 1 - }, - { - "file_name": "images/2.jpeg", - "height": 256, - "width": 256, - "id": 2 - }, - { - "file_name": "images/3.jpeg", - "height": 256, - "width": 256, - "id": 3 - }, - { - "file_name": "images/4.jpeg", - "height": 256, - "width": 256, - "id": 4 - } - ], - "categories": [ - { - "supercategory": "cloth", - "id": 23, - "name": "tshirt", - "keypoints": [ - "neck", - "shoulder1", - "shoulder2" - ] - } - ], - "annotations": [ - { - "category_id": 23, - "id": 1, - "image_id": 1, - "keypoints": [ - 126.0, - 26.6, - 2.0, - 64.1, - 31.8, - 2.0, - 181.8, - 28.9, - 2.0 - ] - }, - { - "category_id": 23, - "id": 2, - "image_id": 2, - "keypoints": [ - 127.68, - 61.3, - 2.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0 - ] - }, - { - "category_id": 23, - "id": 3, - "image_id": 3, - "keypoints": [ - 71.96, - 41.64, - 2.0, - 38.52, - 41.31, - 2.0, - 102.44, - 40.0, - 2.0 - ] - }, - { - "category_id": 23, - "id": 4, - "image_id": 3, - "keypoints": [ - 187.34, - 40.33, - 1.0, - 152.27, - 45.9, - 2.0, - 221.76, - 44.59, - 2.0 - ] - } - ] -} diff --git a/labeling/example/coco_category_configuration.json b/labeling/example/coco_category_configuration.json deleted file mode 100644 index e448053..0000000 --- a/labeling/example/coco_category_configuration.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "categories": [ - { - "name": "tshirt", - "supercategory": "cloth", - "id": 23, - "semantic_types": [ - { - "name": "neck", - "n_keypoints": 1 - }, - { - "name": "shoulder", - "n_keypoints": 2 - } - ] - } - ] -} diff --git a/labeling/example/images/1.jpeg b/labeling/example/images/1.jpeg deleted file mode 100644 index f3f2e14b20f52b012033cc32ba1459bf17e5fb69..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4297 zcmY*c2RPf?`~N0Ehqh*@B3gUKYKT=WEn<&qYqw@?5u+mLxLsn@irOkxRcL9ESRHnx zsy0DuUTUjN^LNMp-rxP4=Q;Cv-_LvAbKd7YhcU?b2Cy6Iq4WR{7yv+r1z`LF*o|+6 zxFC>l#Cf=)f&#!80d#?*$B(lfXFd5R*xA_s1t%vb6be18+1WWcxw)ZGUS8hAiNl7E zj}HbD5D*a&m61_qGyx@t>cL>hA;&)efgwj&n2s`oKmY`ESp5qS6Eg(N!VZ8z5D+uu z7=-22Uy>j&2NQ((+$9S|7BuHAodhMzd%|2Fj=XVxao+_KSRvA#_#i27#fnGyD&jl0 zsIK_OuVN}3hZ}={AuLQs4q5)BI2>}EV>(pAiB3?yrS#&S4wt2faNvhGEBAG~of+=| zj-!7#0|F8Tz#IUC17HjTCk|!79EW{vVEo&^hC*FTh2PKB*4~-kHd7JPwxCIcy|Suw zygad_`1pC|sy#c3?IN?6u#)Tv<@OWa^=d80{F6|TqAs-f^WlTD0tqA;p7iqLdG6xs z!!fNa0Xm|;-ThbjUp+zGz-iJd74Gw__2^=9{v34CrQ9GbDc|FP8WaH|Z5-hdU-hhM zoYG!gvv%2kLaJPx@_yA&G9X`46qdy8l=OJZsNywo->AxGew;50>PNSV^Vc7nC!7h= z*b2Q7Am1oj{}kiXN9l6v!)ZyMs$EiQ!LJGL?3rk&$|f$GT4vUJs!Y$$Na-ACWf4Am z*i36d%jr%b3Hg|~l4*kR09RLrY(bJ}W@f>=YTI>sja;ip#lX98wiSP6J;U*g5iYiU5n0P-Id#dZ?k5PA6vRp;vTQ{Mh7eh|w8z!3s|B(y^ z@a9$R5VGN8_|PeOP)hPE|svO_@(NYdj5h9LVR(HVpw0<^~&!v zUhq2Mv37wz6iPhZbA_mhG1Uk+BYKv2smMjO)YW`UO)T8Y9<0{$7=II2)j6BWoS~{Y zRxP?d9q%+hc;-5C)j>b9IbKXt!1K%|(PvK3tni1FK9AGmjE)aUqiwZXZWGZ^l?|zI zqS=*KUL%+k@H^RnCd%EzDE$*);P|5ec;S4EttRXQH#JnOLUZz|wsfSXNZpoOs#Azu zOxYJF*W!(zRp1#T)vj6}t2yOSNP^yA0+2IK5{vW(DqrWt`Fr~z>}bZvDd722Mb5u0!7UpXtl09FH2 zaHjCSx%WY`{-k%Tx9#mrTw4)u%Dg|2X#V=Rw&fTz@y8A>xxm>!TzbXe#DJhx-<_Xr zwMHm?vEC%hROCpxt-ad^L8&2xUvr}>zHFhzEwx0wQ2nX{=~bK&S*~{Sp|-SBc;Cxh zjB7TkOB_WNof%0ksIv4A#~FPN%sdQ=0=4#j%cRnbiLvzp>B`7hNg_vF-F8LE)%2v_ z+vA@!&lz;2a2R`!rFxFi(y(TOy2lomR;Fyjg_!~S2Vhoq15OmvUFjQl&#g)6OabxA zq9MH>M^80D*`e(fOkoHKiQG)Q622 z(MT&tqxED}et+^VX-m$Mq%x-kD?HNn^_`J}?XW6Ok0t&maazZ2QjARe@jTOA0u}d0 z4n}Fc2yjAWy_d=dmV$8X&AxF);z4r(eYfyK4LF`wE@N)OGM;_3#7B~0dd|=6Zk$d!_&%H}Qe^0^(e+HXFzxxr z9fGmgvo|{544CdQaRM*(9FoEKv|;-=a+ZQHE)5Us5hW|bsknZB0v}z$=9Et`zQDG!{b`+%YF?x1;rw zcXBXS`r&V73UXMNbxdRjbBKxBQgUB6xmE2BeL2Yyy7^A8+#Ri6Z_1WcH0IJP7clB# zWlm#8ws<9?mTvEg3z<^RKbV?Nk34DGAF?x%>DIm+lPV?89pTJOuIAz@)$9x*a;}XP z70^AeD}6}w8D6CZeWzGYHlr_`Pna9^*8lKon96fSF9l7rsy~LdQ@aBT>6KNUtLCb% zi4U=u+E0_W)%33SQwXPbnzSkntD)65^tNX& z?rKT%?Xu3rC{`2`tQBObtqG2B&DM7%6mTZgeG4}RNmnxff@8@fS5fY5o8Zii;MfM$z0Oxr^w_Wl*`HdZDj8#iPtnrs_YD>86H+Mj z;EJK=8xK3qV@hi|qN5F^L{m8J66vnOzoiKv`d%~vAa3MlW!{KaKLcQs4I#a5tC#hU z#vzFb?8e3;LB=TL^QKQ!N?B5HzWbfZ*3oiu0+!36Hs{`|Edz-9(X?UmE6YFX{tb+a z)dTzLa_%irZ`P<-E&_I$mLut`InyM% z$vOS@1$zHa>`qH@kAqzgA9q27OKe9iX5zkec{wu!Xz~qJVU|ITf|1`RVJvzq4 zbm3alOR`)msbyZP>P~X;?1n@B+O5jUPi~Kp0Uum`Qxl_(G_PtjJ&>79S!%co+&#<_ zAZ7tBB@o{Ol(jSufR0u-QsPMNax zk23)6unFR3Hd>#E9X|m7s1ov=75&GY{Bj5f;cSEm(pgMyO8Yw`eIIrqd*-b;oQ9>n}iMhF>5q=HO?feJE+MzK1?^ z69UcqoAvKzpdLH-!qpz*?BqGy>)%*xSr`j#nH1{$@?>- z{|VUZYT}_FcdGD|f;#oqkiMf~OxMX`R^`wLwJaa_yT$gk!f(H7R2 zPOVuN^~oE)X6BiwiI1K;OSlNSc;omp1MpEM5K36F{w>_uPV|U3x~d^Vi&8Qv<1rIM zGxT*TI+eBoZ9(Rw7ii5Uh32)4kB#O+jugh)9R5!J2T|%LQ5$>=p#QZ~R>yw#(}FGj z_<7ZBtp6Nb+nps#bu;@)HaRDb&w78z*`Pz`GJy^SKw?i`+&ud-FxUJg`l2FHfJ|7! zqpqGv2-f?$8#EUYSu^oq9d)J<{)o04F6}j-IO#K<9w6;ExhB7(@g>u7{3Zk(^~CG$ z|5yP42)15RTj#w2p-PC^dEe#Y__3aFOyftEe2*`_IJ}nf!E}-xjiAJ;^stDV^U_&E z*-K|H9bSZ^5Pmz9*z@-(or7AC3rn?PKkQr`J+JJ?;xK;EshDEl0aCp5uLn`WRWr(Z z+cS$sM&IWYKyyW%g)GO3!8lrIzW{3#)?s$$p=oGdkfR2FscP6dRmIr!O>}$+v_N_* z`G=pUNs^C4+EOYuNPh8hC(iCN;QaQ#M+q)N%-n!nlYhud6XE}Gq-Np1!X5QfUEJgr z`N6o-pBcv4gcI^r$M)kZj?K-jYYK$evSh(S2wgjfeHMwHbf+B||Cx!v0Bq-avJ1Jn zu@>vJqY*VB998@OOLsM{W0GLcCXI^THg|j;*{R8Te!Cxez>ia0_|2?_i8dZ>jMXKw kYMM_UEo;knzj3x&@NU>ZoWx8H3xcVdSCs7^dl^Ij2O~J&Qvd(} diff --git a/labeling/example/images/2.jpeg b/labeling/example/images/2.jpeg deleted file mode 100644 index ea5807156d5eeb78a638e3f5f39df6e86040dde5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3034 zcmaixc~p}59>!n5+{vZX^tQQ{;g)GiF4edqCL?ZzVKYGQEM$Ozct0XBf7xVVJ4gyiodB_$;-EiF2Kdx%C> zR`$Q*Hzf<*4uwKR=XNi4&j1nt0>BUu2>5eA65?WD$Traq^4B#0 zK)@g|c>pXz65l3D6_fb=H3%#QQ8;{3QCxkSYxs`y#H*&YQN{xrJD=2{wKcz6tpD-8 zO@xx0iLY{z){$}l$aeR_B?VC?__x$QazS8`hobsP*Q@8n!nYm+3X&iZ7zCsNfE56U z0<+Yal~K7xbEDS(q7=EMGVEsLp|(k{~C5e}4ldQF((j$DOxhPjV9rs2@s=*ZRSQ`PIA8VMdixa-pSyg;L=SodRaE%(y9sB^9EZZs#lSt<^6_ zMEyMUhF+gsre~!VJG-2ZRx(v?`mR)PJo!UE#ceOE9v76qd;PumyjpnIB0ihhGsj#H z&c@E&6{hJ>;tm-Ox(Fyd?zsMq2y{e>|7WWzxYb(CMNgNo1yc6|Hsbk6#xO1%Ib`dM`M|QvzRqiF3ryT1r)a*ajDsQ4jJ-YLU>wR9y2073(dh$lvr1;pvD=4>&dsmx zv9h)`LRP}#x}4q_0fW?s-l)%FK$qBHGY`n@oNQ6;4(%wY7KngjSj_3qKJ=n(mPTo~H zLPkw&rhiVSeR!HK+dS7#Uwhz7M8N$Zx(BU$lhPK5l&Ou2LWH0(`IlF3*2lmbHUpwG z-j1GYPw$uPPB4V&OS&9M2~f%WOvIQPwamA8yus`2OC){o z;u=C{tm9i^zjz(@u-s5eXK{Mi0SEH7fDleRc6^H27gD0ds}DOWs%h!C?($pc9=X9E zF|6EX+(=iN$^_olQw5CFGy?5EDV78mqEA3ocWHrSSGa_|VU7ygPm9#}GCQI+J9~7$ z$W-D+5aF-(I9Eg4yw6!!$ZPSIQL5TJ3aNSYSxK!bQsGR-@R@cDZ|G*m z-B;KtkL-tG0WNOCDaXrp&D^%g%3H&jq?z&^lG_voznW)sk`h0(P6sLqtWnO0-2*~= zzO3TqQ}Zvze9V9k11TMD_EK4I@QHk_cQxr;#CQb9z zw>dwYQcy6x;pU@5qQQbYsH-<87`N!5?PHwlsL)J&nNt0(OO>+K0sO<-2YYOMjvMjP z_T4m}^YPikvSa(X&&*-5jPG-U^{Fl#`t^|DM2{Z`CS(sfRrhkL^n7^{$|9?hw}E?( zPG$FmUDSRGvwwv&3-r{qg@3`swi)7hZC zAmph9?zeD8t}I9<2ld1p$%Uz4p~C{j_c(g>ulLp>(2dxx(Q6I$rds2v76`t~71Agp zB5SCs;#3DN`rS=FkE%q6z5sbV{^Kt|l*abFzK$W;9ZE5r4aXQnm5c@0{@WoLQya8g zy1*mdV?^Gfn?G{WCv3;)RmCvY;KjSsJwFw%g=%+}uNJmaOPF>9yx|j8_~P(Ot>)u8V^RHo(kx5NH+F+7V|Y$ zm8<;8u--yYh<`ZW^019vIMJ&5e3Xp7mGSzlNt}ZzEy^sUd`=I`)Tus0d42YllQb_G zZ%MQ^R|^ZI*?z?4E_8+j#~3&v(YMI2J29NtB&A95M~M@tD*Go@O-}K+*osNAd^{mx zry{O61J@=Ml@V&3*Q?Q3ysCNh&70xj4*pNM{C(rkhdj9)=h&%1#g#X$qv^qgg|HB( zw3B4}Lzar|8p`i++B>thca`> z=-9o(iVt7yuh_g)Zos=}H-#P==d=uzW5a4H9@I5p7!lTroxR@inY$ZvBlwsJcfqi* z+-EhB(Z+m9)IJat!GS^j5kQ^T9RzSflnCmv&R&MC&%DR_TAVrEj0=zFIP?+yLyJ|G zd(I%HVAk6VaoQa|ytq{Eo9T9;Ejo4$Gm*kQ?26Erw3ptkk?vq;*06Ls3Gh4dS9*~K zmrN^saba+)n5xZjlMU{IIW}pmK4w;>^Z9OpjiWq5GZz5%124e9jel==+(v8^l|ywY|q}h7zdFeI_G7aU+V(IES>i{?9>$j<)IW$0q;)dO|BnJ?*K$L*bz7ZDo~Z(?yZ+PkQnGKn1MxfjK0dl?lUYdMImag z_$Pr)-hD$1LUF6tNj|lStAhq$O2yaZGOl3)G5hy=dll|r!H!+3CJ;?6z`@r4sGnb^ zar(0r<%{Wa~puH@ItYeeEUe|hZ?%(m)$Wvv&PMl z$Yi8kcwW%ROlJMWFw%wdZmID!aPaOofxrcLOcXr2yLk;}oRQhoYUq=p3Ch6MUJk{| z4D5VVJ*GT45S1`^5EA*BqWNBJ3($EHZ#N-)FCh1PEnG@i(H37wps1%-vL}1iT}5|V zizS0^v<1wK`S0)R=YCF0i4V;-Kph%VNqe-O{PBg&y6d8K*T3>hsY^>GjI(X$T5eoz dwXwO7yj{7lklYWsB>v2n@Ga&4+azuc{|7n@DRTe- diff --git a/labeling/example/images/3.jpeg b/labeling/example/images/3.jpeg deleted file mode 100644 index 4e9e7d56cc3c72ffe453615245cea4575f20d925..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4498 zcmZ`+c|6o@*Z7<~rdGBPkU02mkn;4Hun z0CXC_W$YZ}s-_{YrYf(bs0h%%0yh9Q4vy0tr#XLv3&iygczAgD`1npN7Z(o?FE1aT zfPlbB;=~CF30=4#EG#ZAAtR$gZv)-~3;-h|69e!M7#UdEn3+zoFfaiBl?MPOMh0ds zfRTZTf%z0OJNqeS*1yj&FfuW7vs^sIqh$8%`Z+&VF=gjB@N7Nt7Jpus1UFZe^0Ekt zoX?3j1_o6Fa<|kfzFR!!%QJtt&HZ=flhS|V-_;oymAEf5@mx1kKIbRqO#cLMvoW0T zV`2~k7`Xu^Zh$@koH^md$bAy)0{t`pkpBN8rfb5;nB5YbQ5B-&{U7N`d0PpcgnjU4 zNl0|zP!mo>jf_RHwz6uF;tDi-cQ22XCHcOtK)pFtI}wsxG&uecg>rCp#kLCtZbMnW zWWJ+lD@4I!QeBhv;)b?@lr@tw_Vhf^3>M ztrxG;aGtx`=i+_~-rDirs+u&ZCLd*x5+>MdWa&rA%l~}z<`a4o52X^JE1ja1sfRYO zWzM8ibrZgeWqOqkfkuldPy7Y_(w+Qm40T@a3z3Y6@9%kY<56%G6I7g^MgUHK{qEKT zeCVyM91j&>eD%LgaM(UmAKecys%vcECK9?Wv%w@BBN1!czYFE_HNZWW5s$tD*yaF!S8i z$1kI;<3_N&P9=IyIhod7`!ydj0e&(>E_r}Lh7M~WhBC)x8Zna&F)%`eot5?jbl%Un zp-~m%3~+~fz@NpZEWhlc~kDDsI}+e3_A|YkW~u1*xEivw;TIY zfKe_=Qn-36f#jRQ89o)ejSq(hrSOT!+-)%W5MQ5$>poZB7moy|ya5QsPn? zQb{EJp=?-d9Utwl8AN!bu4ak!uE>QeXucYwl`qIwyh!Nj>ZevU1yo%=Z8rsmC~%x} z%Ef;l&`B;sjrF7;2+b9Q)Tl~QI=(5;8J--MucKjD2(Nik?&}f@^Yj$zf71LQjj$&> z#i9Rx`RU2aGWGdhz|w(Agf&E|Go-aeKVT@MizaF(!nSL`^HadgCn$v*Indj#KcHZh zX=@Mr(CA9EtXvU%X;S?)1XVq)K8<{$-Nk(~t~m~I`r!qH#F!NGv!>#?40T%oo|59$quIyJs+l4eHT4#w)MJhpaGcV4!w)3H-0uvpe~Gk~1S_tZol0 zD}zKHreTFl?Q;b#&!m{P_HN3Jj#ZlIl`obo@y$toZbpdKIFYU7OC>mF zcX5W5eu-L#`tp%)yjDB1dpCV%yTshz0%ln@`fsnq8H#YOD~8!|Y)tKslm~B=Ug^hK z*wE~AAGqu!E~}P3_Dc#;&k!Y?=lncaTwJ2nS01u?1)kZ&_Nq$3Zu#lifGhwc(j=hk zF-bl294#osBa0;ThpoG|c0t3XD{2(PtzUzVbByx0tnIxE<+N)CLzO+% zv_E8OHN&7S#C+|}R_C%5O=cl6$zg6W=BHe4an<1E^hJa8UA7@}Zyyf zBj5VOtw!R+kfw%@ocv2v+^R$IK_J;G>&ZQnjH1H}NsFN8TuV{!Tc6v-k4mG{$wZ%J>@`n*ec*?6jCMCk_2#++v zPg!~jhD(^Q>g;FN*tPm#TMaO6)>7tav#C|)iw&UKW_3E?Ry+73{{<9=HGDFkws_y} z(F+d^*PK^ZFyL2L08it3Q*zODyjhlQSSdMJD!*N_U>h%JWlmg)p7&~K5AW*{f{}`( zXQ>vs@|Ln!3E!pWmUXN$YaZ`C5gx?)nh=)zkkJxb6|l^Xhixm}mmna4_pYuXV+l6C zEpi2c-~MERlZ!?hYvcV)8?TUKrJlSwvN!Ty>zGXUe;mdXpt=rvS*k;R_|*}qtw5-E zx)_+TaM}s=*B9jGTVm1h`nO#9&9yR%`D0W0olu01{oNULuEWuN{<>sV-#`O39LYzb zB3d5qz!{{h1$$lM^$s!A+(xw(iQHVkbzd5;#`sxN8U^@9KX0KYok%3&S#z?q^wwIw zmAy)QMeNbkd{{}>7x^_gHS4e&waLT{mrR{?J;OZ@1AuKW`r>jQMjb z|B*ukZby4@pw;)z)s^Es);%Wt8VJ^0%h0$xhyx9bf3V3eN?Co|Yi}uMV;zn5uKI?m zufu$^pLl)XeM>6HPaXbE&hhT7c4{S&W^Ykgb+zjfm4IrOkGnDy5nrK^6d#+rp;KJg zZ8|I8@d|BE2TUZ_UUpRbr#fd;V!$MQ!iJpG7Pk7KuL@57U2_sG5>EME6${-EBg!tC z0-ONAdrv_*d_^6!gZO2syREYP>=((YW4XWJMR-gA9cW9$?{V1ewB_@g4L!TYe=mAi zymIADtb4euE5?-Olz2V7P5sO#nsw(vtdRp(t|&TDrMvfjoj3sGUIUm)pX4R1-OzYzO+5J3hm}uD zS>5_<;H`;Dy?GDd^8x@fcZLGWz)W*v{HOZb_`1r0PMwS@SlvjV2F>Q7LZ=n{2$mAm zngoEw40&&Z^7A_5Ey=m#G9<62x&eb}aB9|#hLOrKn)P)WI6B??`q!i(fca#U{~Ju8 zkax6HQ##;hP9v+Zzi8;m%q=iqPHv+E9FgILe>dCdjO^hX*<2iBjBKui*{GPCgsaUe zNjNNmie2YEzM=Tmj>I0?fxQq@y*bTJ2cD&w+PM!10xY&C*WUrhHz;T==FtJ^66%9c z*B`4qbHjU=4^a~o>$D?KMo@=vwtmEC$(&aPApLq6cq5&*czOyIf&=r`1Y(qEHuoF% zT$&Pn&OZ58PyjG+T>G0d0JLPd(}BXl)Wy>2CaXn%McsG0tCe)Xonu<~JB=9=UI;&M zfk!LH<*_jmm2a9>8%MrILJl2|IpMlb#_53E4fVGwL^@z+P6xiwDm33$lX4URj!FQy zQvQ^TKJ@d3kPoaM!6~EZ8uO(Wd`h)Myz$fOU6`^c=})p1T(ZFE;9!0WZmfT1(oQ$f zOU1WWqcRj?wV6`|*+_#3FRk}TuAa2`+HbNWFx(=$<8uzRdAxe?LlmJOl9F1puuT9YMq0OEmG1hiUJrjXL|GHK)(f}Qzx)>oH`U!<#g(`JsYcB9s_{hO0j zyx0_y_C5jPCrY$w+oY%)!=n0_Ryq(R{^bG;?cP#Fk&|RheM5q)> zay3&3zFjSRrB=VgRN0ireonyvN^u;ltLww8GyavRHJ%{#FSlCxX`ZX+4N<;HR^z(N z)#W8%AdK(qBhoa7LAq?Cm>h^x%BTtN*cnIhngwUyFGYhf<}LzcIwFdMu-;<+vD*t z9we9>461*revRE)`88j$HrEd_TZ!gNH!dlB;mY#VEbD|CgD?-^xB7LUHj2eNV5M`W zPbq<7sM2chbM~2|k9XuUp+6#TaK+~HiZYwrImOQQrCW9BPTqXo?3Ad@ x=Ccf|qu|a?6KU1L>*o1vgxh%XaNA4 zvkP!~0=Q!6^u$F`MM_aYN>)Y&a7qT;0bICnf$;((^MBH%ichy;k|iN?6d<=dp12S?YT1@|0WvRa}4L{=r7Qm^_)Aa z{vui$x^w608Lj|mY0lBm(K9g8pJ({b{<9%;Kze>z&;zI2&(8a^YRkR*e3KzLzb4>0 z+a0I?!CAp{^NVYsPObd(kjslgcEK&+8Lo4*bm!>l8R-5?$(ab`r=x!WlJ&psB=_tl zYreB~4fI{W=gm88=LOgmrsV}q3C{~oKLLOj{BGVmLC9-buNnf1+-|^&R1_r9(@cQe*WFP4A z{x`_7i1dfyuEd;>6MMqStydSfB#WYyL_T-1h*B5wtMJ*INUiTKa-6t)bkf09m0~V|M}h!%KM4P(S%&M~{Qz?tW#K-NnFDfV#1kBjwOg$!vAy6tL2` zR2?;p&7(SXT~;Dn5I3IKQU)syvKf2I~ik`TL9--;@)8?T+zUn&{k{cU4hBPUUEl z<2X>10mKui$qKBBwve<)nBEb0_E73LL3wq}tOrw=tg(n4$|>OYdOO0Ez?BiAX!HOB zZAO`%m~%|?1o-%5JX-J=3g^Acn&M zQmyY&*(0rC;uR_EqC+MW*Tg2xHbtd)m`%Oz?GIb9C?&j*l^QXUg~yp*8Izqp6i8R? zc^0@S^`kLsepN_G2W#sA1)DP6-tx1OZqKc)Y!bU$!|3Q|a4pSN*Ze|0X$E1-9J_f8 z%uH9zNmQyC$ulTC%;I(t( zm`AzQt`8#lagjop{x;u0RY%#A+}bX01X=2at?(3&Zld|!Y$7*RGhaN^KH;0zIKLr>!A+*xW8}HPqyNr$dhu#ZmFRiT0w_MjT{1qXtnD7em5R zk7^kOPL-ciO)T>r<`XfhW|b;K(H3@dg2@WP$g)XKQVt~U4Ma6ju@<4~te(A;Ze2q5#Ra{F zr7ESCS|`9v$|DTYp$St#Ix4(hcd_%n+35GLb0mKS_F1->p_ENbO3E-bBvkFSP?bqsVtZe^kr!ER$_4kmu@`-5};lyZQB*aGc3()%k-X1x6DSZsP7Flf8 zT%R#bjBt>X5-5ATqvAU{Z$Aom^~I5WGII%B%~Hy32mZfq?V>lP=cl<&swACv&E<~L zv;v|Xe;#e@A)B}OKy!qi4?9n6t;i2zzq)V4d1Mr-9sEiX4c535cI+00PI8}mCD0vV z#r1^gXOVjl-MxxfSQ-;1pwW@eHH*OUN$gKKCK>=r$obmE5421if=_Ax008LTTIC0n ze&mPPf!L~Pw9kn-Pg_>MZs`zgemVXvb^vG0&RQVxwaQDI$D9Me_MXR7z9li zO0`)r96jzCAz4YPHP@&;Y6=OA`AHt-L%)d4b|;sNB^VkD^EI%I2T>Fye2JG00g>&Dhebl>EK6@l@5lR4A)@abBSGT{ijxzH4cVv^KG z_zE?P>I_y-Onql6YEYnP%-Z4nt;g=Y5$qJe%AL{j-B_s1!`w+U@fuN9Ox4>KH}(!h z;&dx+xauLA*}_v_ERohF=mvR4N9X3Or;`RiJSW|7eLhG}V9W--WKL7k zPL(oCu4U}HAKjg7Y-nsHBuA2u{gQoixYeN`&Q{+IJ!W>t^=m@rcT?es{PYUp-b^!} z*ScIKpbX#f`QqYG^QFTe-ZHvDZzZ|hqp#K{uZ348L;5ZO1!G=oRVRZT<{Jnc z%0U~I4<}4R67|G_15Ju~7(RyPQL*gT)#}4F_N$6!J&9SC6xZXi_K_KK$PlJxc76U3 z%rr7u>;{S4_L`kv*k@UhjhtNCX_%V3l&(Dj=PKxTNL6j(#Mj+dXM?=jAR()_gTJt z+e-DWrJ@mc(|3XoRoB7IOGq}Ky!e)>qfM`4?O&lUNnT~SY5Ld<(SsVQgoo3RUwJ-X zC)Werv>ITdX64ub9NaizJl{f6-Q9ecsIZss!|yG?8~d|O*4P#NQ?)$n@g26nm)fcA|6d zYg5KJyl`RNRYPOO{1S@qp%?K+tNh;i&g;<|F^}YG4d%Y#vDIzxx z6Nx3jRQ|^sR`&g0mJ|E53{r<{`e0S(vGT}d#ij6NOY5aVZ>6=B{z;75JC*l&$ZXx~ zcWvy5O%25kDKjti#O`j(JHoG9ZPzbX1;9juLk3qTaV+9g;ha5NP3W!)dg^L95IMOO zA)E&BvSycD3(`3S>E zcGN8v`&>(Zf* zU+v(g5bELf$${I~MOEiY<&6-Yb^ow^`MS$03f2g>g*CWocwa*xK^+su1BvOp8kb7d z3AdXNrgz@j=4&Zy=phYELKl@>4}@&9OMYcxRayJ1-4@Rg&x1=l=i9m+I!hb+kKOvOPX2FcAwxO9y_D zSob&ds7k7Mp#ne8w|qBhuTbu?BtFP!5`~VOAdMqa>V@&zG3-mikmiW-ybO!teGn%qH=Spqn5y0v1VH^ z57Qhm{LdauV*WII2m1ae@r%cFBYvwpLcLP+xG~AW8?EIloewW2?05}^cBr;j>wB?1 z?dG5Z(%h(Kr_*@~05L&9&SnIWq=>dAR!Sdt=;MUEjIUBl>0b$&Fh7HH;0U!*wAt8tLK^z)nVr_JaVPdsU+>ou53E+f z=&x@^1WKlo`q#Zb-_&h8`tLmaw($u&(t16sK{B7^aQS5nK!GJ^nxbHTZ_@&5C{B;(YwJ`^z`{KBmGm5J{wd|xb$wo zTdNOOyG+L|=LF$NAa#7a^O1VeilgTJq86X@{oqr8#X(@A$a2^0mfsx0@xV8U8r9-* zR@?~;acY15&A5LG=PUPPq(^I$M9m_xZBC>6FSJgmR=++>p##8Z)}yHlkeBy9hyNz@ zo&p|+nQ4e@-ASUH82EQ9?0p0=RGrIBwh-?!DArC)Q$)#e(0{6-V;6r5-~ce$=t&xY zZhO+l8m3B!r%gNSmjUUR)qC&Bcx%Pf8Z7_PSh{;y)b@z9esLdWLpT~4ibQ?-rP3TV zlEFUGBa%8zi1Hi2JAehYJbx%h<(>j;<;XuaWB&1-@J}p6`{Q(2oL5=}C%UK?yq69u z%WE>o+NGMnUqun+m-ARo0e+fNkT<7*k67@P*{Jnq$=CjZ1{0zFbPu=OB1rE*uI@iQ z8ku7)=?XGlK6MeNyTVo@$nim+z7)j#FJ+vDWg-{IqTO@vXd^H(v2!io@>m+^4@yZ% zgU7E}79cV9*d=pIG$Kj;HGUs_a9G??s31bTo&e5e&^qv6apvJzY=HT#_O-l}gn!&2qmu;NwmO#glEXtO)KfrdMj7y)etUryaw~^h37CkaLJFMato}Lr&9jm6-vFL!X|ryWV7_=*OQoGT^`Xyclve{e1uiz)W)k0I0s;WRO-T zS$wl^q|o~VE1gA=+1mwM$ z`n8Ta2l8N-UVrB>et0N4P4e{CTSF}hR? ziayR}IGT>xw5?>|`QeZqTC~kvFGtVpmwckhT(|teB`qf|LY5_dEVV@5d*9>-8H?~0 zGa)x@=_0nLybtj+ht@u_gba4BfTO|qouUBXxIkxM5rt|}aIk>i^g@Cytf{v?7N3%irQuZwHvjZH~AVG~=-AJr-;!OW9t@f5Rzu+WT< z?OqSki_YcnmU{BRPI|_hZ^eVJ)l?F6RVAcU4CrqI(iz1dW^A;2(p}aZ&$XWcL_v&a z_u1a-^6ZxQVvr$P#!)A3^Xx2w`GyC(gj`Rw?zX;IKOUV@$X091N(h|6y3DIAv{UmW zO3VuO1}mjg6jNK>1IY1e;$}%0>X?RZe-=?Ild|d=-iMKB?XQqxHZAv2I)T)027dbU z@}GZq?f;axh;8-K>*iI-4Jhbja%!2`^BI2xHf+#3*+5KVmfI+KHibCdKt#eQrj?U+ zxvr0@vOmwJE}PZn85(vr=Wd5AvGP5T&cwpxBbLG6hcFtrdI7Gatw+?9fw)N+p~~A( zH7l%axXJFIcS+??J?luz=VMabH*CoG;A^d=LZ`_i%|>K~M<7%AFLOKjRWHg;jwYYE zln@uJV}B`zSq;9vsLaDubZ;LWU)ZC%>@~qRhlj)BE%sN-HDE-C2cgp`Bkgyax;b}o zJiFE;72jX|+rD@(AIHTVF|&d#C~L`qv6S?1Ux%aHN#95h69|_B0hjwG1)^Xy-%*&4|`g|Ws2G)h42cEZq(L7VF77jB&cD%b5c!4@mB(l%kDh2ZLO(9 zvp-;zoVbOAJ3@#xq7TeHr8~+udNf9!NVSu2ve((B`^+gL<&$>1b!5lRiTl${XD|7# zhIqusq{luRi7=__^guDvo4f`$sTgn)2)qCgtK)CJ{Uir+J6xp0W)sSXT zHx|}6G9wkcF&A??b`1#pQBB+N^nZrQKY3aiJ@(jvyk*Q|tBL@EXGdsdA~7Ld=xR@= z$C4apW|HcjL7Ea7yjC006T-eNDZFq@=6=-bn;N(R4)nn++m5dkP7Z%<^JqY=S+hH%W;x|_Hb@`ovE{!cZ>pbX-I^dN82|0XQ!vc)c<$UPRITO D*t>61 diff --git a/labeling/file_loading.py b/labeling/file_loading.py deleted file mode 100644 index 823084e..0000000 --- a/labeling/file_loading.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python - -import json - -import xmltodict - - -def get_dict_from_xml(xml_path): - with open(xml_path, "r") as file: - # prefixes @/_ lead to issues with pydantic parsing! so simply use no prefix - xml_dict = xmltodict.parse(file.read(), attr_prefix="") - return xml_dict - - -def get_dict_from_json(path): - with open(path, "r") as file: - return json.load(file) - - -if __name__ == "__main__": - """convert XML to JSON""" - import sys - - assert len(sys.argv) == 2 - xml_path = sys.argv[1] - json_path = xml_path[:-3] + "json" - - print(f"converting {xml_path} to {json_path}") - - xml_dict = get_dict_from_xml(xml_path) - with open(json_path, "w") as outfile: - json.dump(xml_dict, outfile) diff --git a/labeling/parsers/coco_categories_parser.py b/labeling/parsers/coco_categories_parser.py deleted file mode 100644 index c496e77..0000000 --- a/labeling/parsers/coco_categories_parser.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Parser for configuration of COCO categories to convert CVAT XML to COCO Keypoints -""" -from typing import List - -from pydantic import BaseModel - - -class COCOSemanticTypeConfig(BaseModel): - name: str - n_keypoints: int - - -class COCOCategoryConfig(BaseModel): - supercategory: str - id: int - name: str - semantic_types: List[COCOSemanticTypeConfig] - - -class COCOCategoriesConfig(BaseModel): - categories: List[COCOCategoryConfig] diff --git a/labeling/parsers/cvat_keypoints_parser.py b/labeling/parsers/cvat_keypoints_parser.py deleted file mode 100644 index e48a8c1..0000000 --- a/labeling/parsers/cvat_keypoints_parser.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Parser for CVAT Images 1.1 keypoint annotations - -see CVAT Documentation for format information - -This file was created by labeling the example images using the flow that is described in the repo. -Then this xml was converted to JSON using xmltodict -Then an initial version of the parser was created using https://pydantic-docs.helpmanual.io/datamodel_code_generator/: -`datamodel-codegen --input annotations.json --input-file-type json --output cvat_keypoints_parser.py --class-name CVATKeypointsParser` - -And the parser was then further finetuned. - -Note that the attributes cannot have _ as prefix for Pydantic. -""" - -# generated by datamodel-codegen: -# filename: annotations.json -# timestamp: 2022-08-24T12:15:52+00:00 -# then further changed by @tlips - -from __future__ import annotations - -from typing import Any, List, Optional, Union - -from pydantic import BaseModel - - -class Segment(BaseModel): - id: str - start: str - stop: str - url: str - - -class Segments(BaseModel): - segment: Segment - - -class Owner(BaseModel): - username: str - email: str - - -class LabelItem(BaseModel): - name: str - color: str - type: str - attributes: Any - - -class Labels(BaseModel): - label: Union[List[LabelItem], LabelItem] - - -class Task(BaseModel): - id: str - name: str - size: str - mode: str - overlap: str - bugtracker: Any - created: str - updated: str - subset: str - start_frame: str - stop_frame: str - frame_filter: Any - segments: Segments - owner: Owner - assignee: Any - labels: Labels - - -class Meta(BaseModel): - task: Task - dumped: str - - -class Point(BaseModel): - label: str - occluded: str - source: str - points: str - z_order: str - group_id: Optional[str] = "1" # set default group id to 1. - - -class ImageItem(BaseModel): - id: str - name: str - width: str - height: str - points: Optional[Union[List[Point], Point]] = None - - -class Annotations(BaseModel): - version: str - meta: Meta - image: List[ImageItem] - - -class CVATKeypointsParser(BaseModel): - annotations: Annotations diff --git a/labeling/requirements.txt b/labeling/requirements.txt deleted file mode 100644 index efed842..0000000 --- a/labeling/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -xmltodict -pydantic -tqdm diff --git a/labeling/scripts/crop_coco_dataset.py b/labeling/scripts/crop_coco_dataset.py deleted file mode 100644 index 71dc4b9..0000000 --- a/labeling/scripts/crop_coco_dataset.py +++ /dev/null @@ -1,113 +0,0 @@ -import json -import os -from argparse import ArgumentParser -from collections import defaultdict - -import albumentations as A -import cv2 - -from keypoint_detection.data.coco_dataset import COCOKeypointsDataset -from keypoint_detection.data.coco_parser import CocoKeypoints -from keypoint_detection.data.imageloader import ImageLoader, IOSafeImageLoaderDecorator - - -def save_cropped_image_and_edit_annotations( - i, image_info, image_annotations, height_new, width_new, image_loader, input_dataset_path, output_dataset_path -): - input_image_path = os.path.join(input_dataset_path, image_info.file_name) - image = image_loader.get_image(input_image_path, i) - - min_size = min(image.shape[0], image.shape[1]) - transform = A.Compose( - [ - A.CenterCrop(min_size, min_size), - A.Resize(height_new, width_new), - ], - keypoint_params=A.KeypointParams(format="xy", remove_invisible=False), - ) - - # Extract keypoints to the format albumentations wants. - image_keypoints = [] - for annotation in image_annotations: - annotation_keypoints = COCOKeypointsDataset.split_list_in_keypoints(annotation.keypoints) - for keypoint in annotation_keypoints: - image_keypoints.append(keypoint[:2]) - keypoints_xy = [keypoint[:2] for keypoint in image_keypoints] - - # Transform image and keypoints - transformed = transform(image=image, keypoints=keypoints_xy) - transformed_image = transformed["image"] - transformed_keypoints = transformed["keypoints"] - - # Edit the original keypoints. - index = 0 - for annotation in image_annotations: - for i in range(len(annotation.keypoints) // 3): - annotation.keypoints[3 * i : 3 * i + 2] = transformed_keypoints[index] - index += 1 - - # Save transformed image to disk - output_image_path = os.path.join(output_dataset_path, image_info.file_name) - image_directory = os.path.dirname(output_image_path) - os.makedirs(image_directory, exist_ok=True) - image_bgr = cv2.cvtColor(transformed_image, cv2.COLOR_RGB2BGR) - cv2.imwrite(output_image_path, image_bgr) - - -def create_cropped_dataset(input_json_dataset_path, height_new, width_new): - input_dataset_path = os.path.dirname(input_json_dataset_path) - output_dataset_path = input_dataset_path + f"_{height_new}x{width_new}" - - if os.path.exists(output_dataset_path): - print(f"{output_dataset_path} exists, quiting.") - return - - with open(input_json_dataset_path, "r") as file: - data = json.load(file) - parsed_coco = CocoKeypoints(**data) - - image_loader = IOSafeImageLoaderDecorator(ImageLoader()) - annotations = parsed_coco.annotations - - images_annotations = defaultdict(list) - for annotation in annotations: - print(type(annotation)) - images_annotations[annotation.image_id].append(annotation) - - for i, image_info in enumerate(parsed_coco.images): - image_annotations = images_annotations[image_info.id] - save_cropped_image_and_edit_annotations( - i, - image_info, - image_annotations, - height_new, - width_new, - image_loader, - input_dataset_path, - output_dataset_path, - ) - - annotations_json = os.path.join(output_dataset_path, os.path.basename(input_json_dataset_path)) - with open(annotations_json, "w") as file: - json.dump(parsed_coco.dict(exclude_none=True), file) - - return output_dataset_path - - -if __name__ == "__main__": - """ - example usage: - - python crop_coco_dataset.py datasets/towel_testset_0 256 256 - - This will create a new dataset called towel_testset_0_256x256 in the same directory as the old one. - The old dataset will be unaltered. - Currently only square outputs are supported. - """ - - parser = ArgumentParser() - parser.add_argument("input_json_dataset_path") - parser.add_argument("height_new", type=int) - parser.add_argument("width_new", type=int) - args = parser.parse_args() - create_cropped_dataset(**vars(args)) From d73dbaf2be72154b73adb46c9765a87b41bf453a Mon Sep 17 00:00:00 2001 From: tlpss Date: Wed, 30 Aug 2023 12:43:51 +0200 Subject: [PATCH 24/45] log mAP over channels for all threshold distances --- keypoint_detection/models/detector.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index 78cb119..5c4cf76 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -337,14 +337,22 @@ def test_step(self, test_batch, batch_idx): self.log("test/gt_loss", result_dict["gt_loss"]) def log_and_reset_mean_ap(self, mode: str): - mean_ap = 0.0 + mean_ap_per_threshold = torch.zeros(len(self.maximal_gt_keypoint_pixel_distances)) metrics = self.ap_test_metrics if mode == "test" else self.ap_validation_metrics for channel_idx, channel_name in enumerate(self.keypoint_channel_configuration): - channel_mean_ap = self.compute_and_log_metrics_for_channel(metrics[channel_idx], channel_name, mode) - mean_ap += channel_mean_ap - mean_ap /= len(self.keypoint_channel_configuration) + channel_aps = self.compute_and_log_metrics_for_channel(metrics[channel_idx], channel_name, mode) + mean_ap_per_threshold += torch.tensor(channel_aps) + + for i, maximal_distance in enumerate(self.maximal_gt_keypoint_pixel_distances): + self.log( + f"{mode}/meanAP/d={float(maximal_distance):.1f}", + mean_ap_per_threshold[i] / len(self.keypoint_channel_configuration), + ) + + mean_ap = mean_ap_per_threshold.mean() / len(self.keypoint_channel_configuration) self.log(f"{mode}/meanAP", mean_ap) + self.log(f"{mode}/meanAP/meanAP", mean_ap) def validation_epoch_end(self, outputs): """ @@ -386,9 +394,9 @@ def update_channel_ap_metrics( def compute_and_log_metrics_for_channel( self, metrics: KeypointAPMetrics, channel: str, training_mode: str - ) -> float: + ) -> List[float]: """ - logs AP of predictions of single Channel² for each threshold distance (as configured) for the categorization of the keypoints into a confusion matrix. + logs AP of predictions of single Channel for each threshold distance (as configured) for the categorization of the keypoints into a confusion matrix. Also resets metric and returns resulting meanAP over all channels. """ # compute ap's @@ -399,9 +407,9 @@ def compute_and_log_metrics_for_channel( mean_ap = sum(ap_metrics.values()) / len(ap_metrics.values()) - self.log(f"{training_mode}/{channel}_meanAP", mean_ap) # log top level for wandb hyperparam chart. + self.log(f"{training_mode}/{channel}_ap/meanAP", mean_ap) # log top level for wandb hyperparam chart. metrics.reset() - return mean_ap + return list(ap_metrics.values()) def is_ap_epoch(self) -> bool: """Returns True if the AP should be calculated in this epoch.""" From 65be243ebf2825ae79faacaaa2c3ede08444a92f Mon Sep 17 00:00:00 2001 From: tlpss Date: Wed, 30 Aug 2023 14:30:06 +0200 Subject: [PATCH 25/45] calculate & log AP for training --- keypoint_detection/models/detector.py | 36 ++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index 5c4cf76..be8a413 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -120,6 +120,9 @@ def __init__( ] self.maximal_gt_keypoint_pixel_distances = maximal_gt_keypoint_pixel_distances + self.ap_training_metrics = [ + KeypointAPMetrics(self.maximal_gt_keypoint_pixel_distances) for _ in self.keypoint_channel_configuration + ] self.ap_validation_metrics = [ KeypointAPMetrics(self.maximal_gt_keypoint_pixel_distances) for _ in self.keypoint_channel_configuration ] @@ -251,8 +254,17 @@ def shared_step(self, batch, batch_idx, include_visualization_data_in_result_dic def training_step(self, train_batch, batch_idx): log_images = batch_idx == 0 and self.current_epoch > 0 + should_log_ap = ( + self.is_ap_epoch() + ) # and batch_idx < 20 # limit AP calculation to first 20 batches to save time + include_vis_data = log_images or should_log_ap + + result_dict = self.shared_step( + train_batch, batch_idx, include_visualization_data_in_result_dict=include_vis_data + ) - result_dict = self.shared_step(train_batch, batch_idx, include_visualization_data_in_result_dict=log_images) + if should_log_ap: + self.update_ap_metrics(result_dict, self.ap_training_metrics) if log_images: image_grids = self.visualize_predictions_channels(result_dict) @@ -340,20 +352,32 @@ def log_and_reset_mean_ap(self, mode: str): mean_ap_per_threshold = torch.zeros(len(self.maximal_gt_keypoint_pixel_distances)) metrics = self.ap_test_metrics if mode == "test" else self.ap_validation_metrics + # calculate APs for each channel and each threshold distance, and log them + print(f" # {mode} metrics:") for channel_idx, channel_name in enumerate(self.keypoint_channel_configuration): channel_aps = self.compute_and_log_metrics_for_channel(metrics[channel_idx], channel_name, mode) mean_ap_per_threshold += torch.tensor(channel_aps) + # calculate the mAP over all channels for each threshold distance, and log them for i, maximal_distance in enumerate(self.maximal_gt_keypoint_pixel_distances): self.log( f"{mode}/meanAP/d={float(maximal_distance):.1f}", mean_ap_per_threshold[i] / len(self.keypoint_channel_configuration), ) + # calculate the mAP over all channels and all threshold distances, and log it mean_ap = mean_ap_per_threshold.mean() / len(self.keypoint_channel_configuration) self.log(f"{mode}/meanAP", mean_ap) self.log(f"{mode}/meanAP/meanAP", mean_ap) + def training_epoch_end(self, outputs): + """ + Called on the end of a training epoch. + Used to compute and log the AP metrics. + """ + if self.is_ap_epoch(): + self.log_and_reset_mean_ap("train") + def validation_epoch_end(self, outputs): """ Called on the end of a validation epoch. @@ -396,18 +420,18 @@ def compute_and_log_metrics_for_channel( self, metrics: KeypointAPMetrics, channel: str, training_mode: str ) -> List[float]: """ - logs AP of predictions of single Channel for each threshold distance (as configured) for the categorization of the keypoints into a confusion matrix. - Also resets metric and returns resulting meanAP over all channels. + logs AP of predictions of single Channel for each threshold distance. + Also resets metric and returns resulting AP for all distances. """ - # compute ap's ap_metrics = metrics.compute() - print(f"{ap_metrics=}") + rounded_ap_metrics = {k: round(v, 3) for k, v in ap_metrics.items()} + print(f"{channel} : {rounded_ap_metrics}") for maximal_distance, ap in ap_metrics.items(): self.log(f"{training_mode}/{channel}_ap/d={float(maximal_distance):.1f}", ap) mean_ap = sum(ap_metrics.values()) / len(ap_metrics.values()) - self.log(f"{training_mode}/{channel}_ap/meanAP", mean_ap) # log top level for wandb hyperparam chart. + metrics.reset() return list(ap_metrics.values()) From 918a60a79719c7d23ca79df28e05a767f3aa4907 Mon Sep 17 00:00:00 2001 From: Thomas Lips <37955681+tlpss@users.noreply.github.com> Date: Wed, 30 Aug 2023 22:54:07 +0200 Subject: [PATCH 26/45] Fiftyone viewer (#33) * initial version of the fityone viewer * continue on fiftyone viewer * avoid loading coco annotations * move fiftyone viewer * add mAP to fiftyone viewer * refactor & configure colors * refactor --- keypoint_detection/train/train.py | 11 +- keypoint_detection/train/utils.py | 9 +- scripts/fiftyone_viewer.py | 228 ++++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 239 insertions(+), 10 deletions(-) create mode 100644 scripts/fiftyone_viewer.py diff --git a/keypoint_detection/train/train.py b/keypoint_detection/train/train.py index 9dfbe1d..e703a80 100644 --- a/keypoint_detection/train/train.py +++ b/keypoint_detection/train/train.py @@ -1,5 +1,5 @@ from argparse import ArgumentParser -from typing import List, Tuple +from typing import Tuple import pytorch_lightning as pl import wandb @@ -9,7 +9,7 @@ from keypoint_detection.data.datamodule import KeypointsDataModule from keypoint_detection.models.backbones.backbone_factory import BackboneFactory from keypoint_detection.models.detector import KeypointDetector -from keypoint_detection.train.utils import create_pl_trainer +from keypoint_detection.train.utils import create_pl_trainer, parse_channel_configuration from keypoint_detection.utils.path import get_wandb_log_dir_path @@ -105,13 +105,6 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: return model, trainer -def parse_channel_configuration(channel_configuration: str) -> List[List[str]]: - assert isinstance(channel_configuration, str) - channels = channel_configuration.split(";") - channels = [[category.strip() for category in channel.split("=")] for channel in channels] - return channels - - if __name__ == "__main__": """ 1. creates argumentparser with Model, Trainer and system paramaters; which can be used to overwrite default parameters diff --git a/keypoint_detection/train/utils.py b/keypoint_detection/train/utils.py index 5ce8493..f909de6 100644 --- a/keypoint_detection/train/utils.py +++ b/keypoint_detection/train/utils.py @@ -1,5 +1,5 @@ import inspect -from typing import Optional, Tuple +from typing import List, Optional, Tuple import pytorch_lightning as pl import torch @@ -93,3 +93,10 @@ def create_pl_trainer(hparams: dict, wandb_logger: WandbLogger) -> Trainer: trainer = pl.Trainer(**trainer_kwargs, callbacks=[early_stopping, checkpoint_callback]) return trainer + + +def parse_channel_configuration(channel_configuration: str) -> List[List[str]]: + assert isinstance(channel_configuration, str) + channels = channel_configuration.split(";") + channels = [[category.strip() for category in channel.split("=")] for channel in channels] + return channels diff --git a/scripts/fiftyone_viewer.py b/scripts/fiftyone_viewer.py new file mode 100644 index 0000000..e6f49d9 --- /dev/null +++ b/scripts/fiftyone_viewer.py @@ -0,0 +1,228 @@ +"""use fiftyone to visualize the predictions of trained keypoint detectors on a dataset. Very useful for debugging and understanding the models predictions.""" +import os +from collections import defaultdict +from typing import List, Optional, Tuple + +import fiftyone as fo +import numpy as np +import torch +import tqdm + +from keypoint_detection.data.coco_dataset import COCOKeypointsDataset +from keypoint_detection.models.detector import KeypointDetector +from keypoint_detection.models.metrics import DetectedKeypoint, Keypoint, KeypointAPMetrics +from keypoint_detection.train.utils import parse_channel_configuration +from keypoint_detection.utils.heatmap import compute_keypoint_probability, get_keypoints_from_heatmap_batch_maxpool +from keypoint_detection.utils.load_checkpoints import get_model_from_wandb_checkpoint + + +class DetectorFiftyoneViewer: + def __init__( + self, + dataset_path: str, + models: dict[str, KeypointDetector], + channel_config: str, + detect_only_visible_keypoints: bool = False, + n_samples: Optional[int] = None, + ): + self.dataset_path = dataset_path + self.models = models + self.channel_config = channel_config + self.detect_only_visible_keypoints = detect_only_visible_keypoints + self.n_samples = n_samples + self.parsed_channel_config = parse_channel_configuration(channel_config) + + dataset = COCOKeypointsDataset( + dataset_path, self.parsed_channel_config, detect_only_visible_keypoints=detect_only_visible_keypoints + ) + if self.n_samples is not None: + dataset = torch.utils.data.Subset(dataset, range(0, self.n_samples)) + + self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=0) + + # create the AP metrics + self.ap_metrics = { + name: [KeypointAPMetrics(model.maximal_gt_keypoint_pixel_distances) for _ in self.parsed_channel_config] + for name, model in models.items() + } + + # set all models to eval mode to be sure. + for model in self.models.values(): + model.eval() + + self.predicted_keypoints = {model_name: [] for model_name in models.keys()} + self.gt_keypoints = [] + # {model: {sample_idx: {channel_idx: [ap_score]}} + self.ap_scores = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + + def predict_and_compute_metrics(self): + with torch.no_grad(): + sample_idx = 0 + for image, keypoints in tqdm.tqdm(self.dataloader): + # [[channel1], [[[x,y],[x,y]]] + gt_keypoints = [] + for channel in keypoints: + gt_keypoints.append([[kp[0].item(), kp[1].item()] for kp in channel]) + self.gt_keypoints.append(gt_keypoints) + + for model_name, model in self.models.items(): + heatmaps = model(image)[0] + # extract keypoints from heatmaps for each channel + predicted_keypoints = get_keypoints_from_heatmap_batch_maxpool(heatmaps.unsqueeze(0))[0] + predicted_keypoint_probabilities = [ + compute_keypoint_probability(heatmaps[i], predicted_keypoints[i]) for i in range(len(heatmaps)) + ] + self.predicted_keypoints[model_name].append( + [predicted_keypoints, predicted_keypoint_probabilities] + ) + + #### METRIC COMPUTATION #### + for metric in self.ap_metrics[model_name]: + metric.reset() + + for channel_idx in range(len(self.parsed_channel_config)): + metric_detected_keypoints = predicted_keypoints[channel_idx] + probabilities = predicted_keypoint_probabilities[channel_idx] + metric_detected_keypoints = [ + DetectedKeypoint(kp[0], kp[1], p) + for kp, p in zip(metric_detected_keypoints, probabilities) + ] + metric_gt_formatted_keypoints = [Keypoint(kp[0], kp[1]) for kp in gt_keypoints[channel_idx]] + self.ap_metrics[model_name][channel_idx].update( + metric_detected_keypoints, metric_gt_formatted_keypoints + ) + + for channel_idx in range(len(self.parsed_channel_config)): + self.ap_scores[model_name][sample_idx].update( + {channel_idx: list(self.ap_metrics[model_name][channel_idx].compute().values())} + ) + + sample_idx += 1 + + def visualize_predictions( + self, + ): + """visualize keypoint detectors on a coco dataset. Requires the coco json, thechannel config and a dict of wandb checkpoints.""" + + # iterate over dataset and compute predictions & gt keypoints as well as metrics + + ## create fiftyone dataset and add the predictions and gt + + fo_dataset = fo.Dataset.from_dir( + dataset_type=fo.types.COCODetectionDataset, + data_path=os.path.dirname(self.dataset_path), + label_types=[], # do not load the coco annotations + labels_path=self.dataset_path, + ) + + fo_dataset.add_dynamic_sample_fields() + + fo_dataset = fo_dataset.limit(self.n_samples) + + # add the ground truth to the dataset + for sample_idx, sample in enumerate(fo_dataset): + self._add_instance_keypoints_to_fo_sample( + sample, "ground_truth_keypoints", self.gt_keypoints[sample_idx], None, self.parsed_channel_config + ) + + # add the predictions to the dataset + for model_name, model in self.models.items(): + for sample_idx, sample in enumerate(fo_dataset): + keypoints, probabilities = self.predicted_keypoints[model_name][sample_idx] + self._add_instance_keypoints_to_fo_sample( + sample, f"{model_name}_keypoints", keypoints, probabilities, self.parsed_channel_config + ) + model_ap_scores = self.ap_scores[model_name][sample_idx] + + # log map + ap_values = np.zeros((len(self.parsed_channel_config), len(model.maximal_gt_keypoint_pixel_distances))) + for channel_idx in range(len(self.parsed_channel_config)): + for max_dist_idx in range(len(model.maximal_gt_keypoint_pixel_distances)): + ap_values[channel_idx, max_dist_idx] = model_ap_scores[channel_idx][max_dist_idx] + sample[f"{model_name}_keypoints_mAP"] = ap_values.mean() + sample.save() + # could do only one loop instead of two for the predictions usually, but we have to compute the GT keypoints, so we need to loop over the dataset anyway + # https://docs.voxel51.com/user_guide/dataset_creation/index.html#model-predictions + + print(fo_dataset) + + session = fo.launch_app(dataset=fo_dataset) + session = self._configure_session_colors(session) + session.wait() + + def _configure_session_colors(self, session: fo.Session) -> fo.Session: + """ + set colors such that each model has a different color and the mAP labels have the same color as the keypoints. + """ + + # chatgpt color pool + color_pool = [ + "#FF00FF", # Neon Purple + "#00FF00", # Electric Green + "#FFFF00", # Cyber Yellow + "#0000FF", # Laser Blue + "#FF0000", # Radioactive Red + "#00FFFF", # Galactic Teal + "#FF00AA", # Quantum Pink + "#C0C0C0", # Holographic Silver + "#000000", # Abyssal Black + "#FFA500", # Cosmic Orange + ] + color_fields = [] + color_fields.append({"path": "ground_truth_keypoints", "fieldColor": color_pool[-1]}) + for model_idx, model_name in enumerate(self.models.keys()): + color_fields.append({"path": f"{model_name}_keypoints", "fieldColor": color_pool[model_idx]}) + color_fields.append({"path": f"{model_name}_keypoints_mAP", "fieldColor": color_pool[model_idx]}) + session.color_scheme = fo.ColorScheme(color_pool=color_pool, fields=color_fields) + return session + + def _add_instance_keypoints_to_fo_sample( + self, + sample, + predictions_name, + instance_keypoints: List[List[Tuple]], + keypoint_probabilities: List[List[float]], + parsed_channels: List[List[str]], + ) -> fo.Sample: + """adds the detected keypoints to the sample in the fiftyone format""" + assert len(instance_keypoints) == len(parsed_channels) + # assert instance_keypoints[0][0][0] > 1.0 # check if the keypoints are not normalized yet + fo_keypoints = [] + for channel_idx in range(len(instance_keypoints)): + channel_keypoints = instance_keypoints[channel_idx] + # normalize the keypoints to the image size + width = sample["metadata"]["width"] + height = sample["metadata"]["height"] + channel_keypoints = [[kp[0] / width, kp[1] / height] for kp in channel_keypoints] + if keypoint_probabilities is not None: + channel_keypoint_probabilities = keypoint_probabilities[channel_idx] + else: + channel_keypoint_probabilities = None + fo_keypoints.append( + fo.Keypoint( + label="=".join(parsed_channels[channel_idx]), + points=channel_keypoints, + confidence=channel_keypoint_probabilities, + ) + ) + + sample[predictions_name] = fo.Keypoints(keypoints=fo_keypoints) + sample.save() + return sample + + +if __name__ == "__main__": + # TODO: make CLI for this -> hydra config? + checkpoint_dict = { + "maxvit-256-flat": "tlips/synthetic-cloth-keypoints-quest-for-precision/model-5ogj44k0:v0", + "maxvit-512-flat": "tlips/synthetic-cloth-keypoints-quest-for-precision/model-1of5e6qs:v0", + } + + dataset_path = "/storage/users/tlips/RTFClothes/towels-test_resized_256x256/towels-test.json" + channel_config = "corner0=corner1=corner2=corner3" + detect_only_visible_keypoints = False + n_samples = 100 + models = {key: get_model_from_wandb_checkpoint(value) for key, value in checkpoint_dict.items()} + visualizer = DetectorFiftyoneViewer(dataset_path, models, channel_config, detect_only_visible_keypoints, n_samples) + visualizer.predict_and_compute_metrics() + visualizer.visualize_predictions() diff --git a/setup.py b/setup.py index 8cc848e..98521e4 100644 --- a/setup.py +++ b/setup.py @@ -24,5 +24,6 @@ # for labeling package, should be moved in time to separate setup.py "xmltodict", "pydantic", + "fiftyone", ], ) From 8085dbba1c2bc30a712f08db712daf0bc1ecea77 Mon Sep 17 00:00:00 2001 From: tlpss Date: Thu, 31 Aug 2023 12:53:22 +0200 Subject: [PATCH 27/45] fix bug in logging train AP and reduce # batches for which train AP is calculated to limit memory usage --- keypoint_detection/models/detector.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index be8a413..29cd20c 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -254,9 +254,7 @@ def shared_step(self, batch, batch_idx, include_visualization_data_in_result_dic def training_step(self, train_batch, batch_idx): log_images = batch_idx == 0 and self.current_epoch > 0 - should_log_ap = ( - self.is_ap_epoch() - ) # and batch_idx < 20 # limit AP calculation to first 20 batches to save time + should_log_ap = self.is_ap_epoch() and batch_idx < 20 # limit AP calculation to first 20 batches to save time include_vis_data = log_images or should_log_ap result_dict = self.shared_step( @@ -328,7 +326,7 @@ def validation_step(self, val_batch, batch_idx): if self.is_ap_epoch(): self.update_ap_metrics(result_dict, self.ap_validation_metrics) - log_images = batch_idx == 0 and self.current_epoch > 0 + log_images = batch_idx == 0 and self.current_epoch > 0 and self.is_ap_epoch() if log_images: image_grids = self.visualize_predictions_channels(result_dict) self.log_image_grids(image_grids, mode="validation") @@ -350,7 +348,14 @@ def test_step(self, test_batch, batch_idx): def log_and_reset_mean_ap(self, mode: str): mean_ap_per_threshold = torch.zeros(len(self.maximal_gt_keypoint_pixel_distances)) - metrics = self.ap_test_metrics if mode == "test" else self.ap_validation_metrics + if mode == "train": + metrics = self.ap_training_metrics + elif mode == "validation": + metrics = self.ap_validation_metrics + elif mode == "test": + metrics = self.ap_test_metrics + else: + raise ValueError(f"mode {mode} not recognized") # calculate APs for each channel and each threshold distance, and log them print(f" # {mode} metrics:") From 21d78dfe0a4bb3334c63f09458239d27a4743283 Mon Sep 17 00:00:00 2001 From: tlpss Date: Thu, 31 Aug 2023 14:40:18 +0200 Subject: [PATCH 28/45] add Maxvit pico encoder --- .../models/backbones/backbone_factory.py | 12 +++- .../models/backbones/maxvit_unet.py | 55 +++++++++++-------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/keypoint_detection/models/backbones/backbone_factory.py b/keypoint_detection/models/backbones/backbone_factory.py index 6ecf5e0..05b60bd 100644 --- a/keypoint_detection/models/backbones/backbone_factory.py +++ b/keypoint_detection/models/backbones/backbone_factory.py @@ -4,7 +4,7 @@ from keypoint_detection.models.backbones.base_backbone import Backbone from keypoint_detection.models.backbones.convnext_unet import ConvNeXtUnet from keypoint_detection.models.backbones.dilated_cnn import DilatedCnn -from keypoint_detection.models.backbones.maxvit_unet import MaxVitUnet +from keypoint_detection.models.backbones.maxvit_unet import MaxVitPicoUnet, MaxVitUnet from keypoint_detection.models.backbones.mobilenetv3 import MobileNetV3 from keypoint_detection.models.backbones.s3k import S3K from keypoint_detection.models.backbones.unet import Unet @@ -12,7 +12,15 @@ class BackboneFactory: # TODO: how to auto-register with __init__subclass over multiple files? - registered_backbone_classes: List[Backbone] = [Unet, ConvNeXtUnet, MaxVitUnet, S3K, DilatedCnn, MobileNetV3] + registered_backbone_classes: List[Backbone] = [ + Unet, + ConvNeXtUnet, + MaxVitUnet, + MaxVitPicoUnet, + S3K, + DilatedCnn, + MobileNetV3, + ] @staticmethod def create_backbone(backbone_type: str, **kwargs) -> Backbone: diff --git a/keypoint_detection/models/backbones/maxvit_unet.py b/keypoint_detection/models/backbones/maxvit_unet.py index fccdcf8..12d302c 100644 --- a/keypoint_detection/models/backbones/maxvit_unet.py +++ b/keypoint_detection/models/backbones/maxvit_unet.py @@ -37,30 +37,31 @@ class MaxVitUnet(Backbone): stage 4 ---1/32----| """ - # manually gathered for maxvit_nano_rw_256 - feature_config = [ + # 15M params + FEATURE_CONFIG = [ {"down": 2, "channels": 64}, {"down": 4, "channels": 64}, {"down": 8, "channels": 128}, {"down": 16, "channels": 256}, {"down": 32, "channels": 512}, ] + MODEL_NAME = "maxvit_nano_rw_256" feature_layers = ["stem", "stages.0", "stages.1", "stages.2", "stages.3"] def __init__(self, **kwargs) -> None: super().__init__() - self.encoder = timm.create_model("maxvit_nano_rw_256", pretrained=True, num_classes=0) # 15M params + self.encoder = timm.create_model(self.MODEL_NAME, pretrained=True, num_classes=0) self.feature_extractor = create_feature_extractor(self.encoder, self.feature_layers) self.decoder_blocks = nn.ModuleList() - for config_skip, config_in in zip(self.feature_config, self.feature_config[1:]): + for config_skip, config_in in zip(self.FEATURE_CONFIG, self.FEATURE_CONFIG[1:]): block = UpSamplingBlock(config_in["channels"], config_skip["channels"], config_skip["channels"], 3) self.decoder_blocks.append(block) self.final_conv = nn.Conv2d( - self.feature_config[0]["channels"], self.feature_config[0]["channels"], 3, padding="same" + self.FEATURE_CONFIG[0]["channels"], self.FEATURE_CONFIG[0]["channels"], 3, padding="same" ) self.final_upsampling_block = UpSamplingBlock( - self.feature_config[0]["channels"], 3, self.feature_config[0]["channels"], 3 + self.FEATURE_CONFIG[0]["channels"], 3, self.FEATURE_CONFIG[0]["channels"], 3 ) def forward(self, x): @@ -76,26 +77,36 @@ def forward(self, x): return x def get_n_channels_out(self): - return self.feature_config[0]["channels"] + return self.FEATURE_CONFIG[0]["channels"] + + +class MaxVitPicoUnet(MaxVitUnet): + MODEL_NAME = "maxvit_rmlp_pico_rw_256" # 7.5M params. + FEATURE_CONFIG = [ + {"down": 2, "channels": 32}, + {"down": 4, "channels": 32}, + {"down": 8, "channels": 64}, + {"down": 16, "channels": 128}, + {"down": 32, "channels": 256}, + ] if __name__ == "__main__": - # model = timm.create_model("maxvit_rmlp_pico_rw_256") + model = timm.create_model("maxvit_rmlp_pico_rw_256") # model = timm.create_model("maxvit_nano_rw_256") - # feature_extractor = create_feature_extractor(model, ["stem", "stages.0", "stages.1", "stages.2", "stages.3"]) - # x = torch.zeros((1, 3, 256, 256)) - # features = list(feature_extractor(x).values()) - # n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) - # print(f"num params = {n_params/10**6:.2f} M") - # feature_config = [] - # for x in features: - # print(f"{x.shape=}") - # config = {"down": 256 // x.shape[2], "channels": x.shape[1]} - # feature_config.append(config) - # print(f"{feature_config=}") - - print("creating MaxViTUnet") - model = MaxVitUnet() + feature_extractor = create_feature_extractor(model, ["stem", "stages.0", "stages.1", "stages.2", "stages.3"]) + x = torch.zeros((1, 3, 256, 256)) + features = list(feature_extractor(x).values()) + n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + print(f"num params = {n_params/10**6:.2f} M") + feature_config = [] + for x in features: + print(f"{x.shape=}") + config = {"down": 256 // x.shape[2], "channels": x.shape[1]} + feature_config.append(config) + print(f"{feature_config=}") + + model = MaxVitPicoUnet() x = torch.zeros((1, 3, 256, 256)) y = model(x) print(f"{y.shape=}") From f98598b82aaa1bf5525269f0850518f923c61c02 Mon Sep 17 00:00:00 2001 From: tlpss Date: Thu, 31 Aug 2023 14:41:08 +0200 Subject: [PATCH 29/45] manually set threshold distances in fiftyone viewer --- scripts/fiftyone_viewer.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/fiftyone_viewer.py b/scripts/fiftyone_viewer.py index e6f49d9..828a584 100644 --- a/scripts/fiftyone_viewer.py +++ b/scripts/fiftyone_viewer.py @@ -24,6 +24,7 @@ def __init__( channel_config: str, detect_only_visible_keypoints: bool = False, n_samples: Optional[int] = None, + ap_threshold_distances: Optional[List[int]] = None, ): self.dataset_path = dataset_path self.models = models @@ -31,6 +32,11 @@ def __init__( self.detect_only_visible_keypoints = detect_only_visible_keypoints self.n_samples = n_samples self.parsed_channel_config = parse_channel_configuration(channel_config) + self.ap_threshold_distances = ap_threshold_distances + if self.ap_threshold_distances is None: + self.ap_threshold_distances = [ + 2, + ] dataset = COCOKeypointsDataset( dataset_path, self.parsed_channel_config, detect_only_visible_keypoints=detect_only_visible_keypoints @@ -42,8 +48,8 @@ def __init__( # create the AP metrics self.ap_metrics = { - name: [KeypointAPMetrics(model.maximal_gt_keypoint_pixel_distances) for _ in self.parsed_channel_config] - for name, model in models.items() + name: [KeypointAPMetrics(self.ap_threshold_distances) for _ in self.parsed_channel_config] + for name in models.keys() } # set all models to eval mode to be sure. @@ -135,9 +141,9 @@ def visualize_predictions( model_ap_scores = self.ap_scores[model_name][sample_idx] # log map - ap_values = np.zeros((len(self.parsed_channel_config), len(model.maximal_gt_keypoint_pixel_distances))) + ap_values = np.zeros((len(self.parsed_channel_config), len(self.ap_threshold_distances))) for channel_idx in range(len(self.parsed_channel_config)): - for max_dist_idx in range(len(model.maximal_gt_keypoint_pixel_distances)): + for max_dist_idx in range(len(self.ap_threshold_distances)): ap_values[channel_idx, max_dist_idx] = model_ap_scores[channel_idx][max_dist_idx] sample[f"{model_name}_keypoints_mAP"] = ap_values.mean() sample.save() @@ -214,14 +220,18 @@ def _add_instance_keypoints_to_fo_sample( if __name__ == "__main__": # TODO: make CLI for this -> hydra config? checkpoint_dict = { - "maxvit-256-flat": "tlips/synthetic-cloth-keypoints-quest-for-precision/model-5ogj44k0:v0", - "maxvit-512-flat": "tlips/synthetic-cloth-keypoints-quest-for-precision/model-1of5e6qs:v0", + # "maxvit-256-flat": "tlips/synthetic-cloth-keypoints-quest-for-precision/model-5ogj44k0:v0", + # "maxvit-512-flat": "tlips/synthetic-cloth-keypoints-quest-for-precision/model-1of5e6qs:v0", + "maxvit-pyflex-20k": "tlips/synthetic-cloth-keypoints/model-qiellxgb:v0" } dataset_path = "/storage/users/tlips/RTFClothes/towels-test_resized_256x256/towels-test.json" + dataset_path = ( + "/home/tlips/Documents/synthetic-cloth-data/synthetic-cloth-data/data/datasets/TOWEL/00/annotations_val.json" + ) channel_config = "corner0=corner1=corner2=corner3" detect_only_visible_keypoints = False - n_samples = 100 + n_samples = 50 models = {key: get_model_from_wandb_checkpoint(value) for key, value in checkpoint_dict.items()} visualizer = DetectorFiftyoneViewer(dataset_path, models, channel_config, detect_only_visible_keypoints, n_samples) visualizer.predict_and_compute_metrics() From ddc27df645298a05966730b94e512ed7cca137c2 Mon Sep 17 00:00:00 2001 From: tlpss Date: Thu, 31 Aug 2023 18:38:05 +0200 Subject: [PATCH 30/45] fix bug in association between GT keypoints and fiftyone samples --- scripts/fiftyone_viewer.py | 65 ++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/scripts/fiftyone_viewer.py b/scripts/fiftyone_viewer.py index 828a584..bfc7660 100644 --- a/scripts/fiftyone_viewer.py +++ b/scripts/fiftyone_viewer.py @@ -38,13 +38,9 @@ def __init__( 2, ] - dataset = COCOKeypointsDataset( + self.coco_dataset = COCOKeypointsDataset( dataset_path, self.parsed_channel_config, detect_only_visible_keypoints=detect_only_visible_keypoints ) - if self.n_samples is not None: - dataset = torch.utils.data.Subset(dataset, range(0, self.n_samples)) - - self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=0) # create the AP metrics self.ap_metrics = { @@ -61,14 +57,36 @@ def __init__( # {model: {sample_idx: {channel_idx: [ap_score]}} self.ap_scores = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + # create the fiftyone dataset + self.fo_dataset = fo.Dataset.from_dir( + dataset_type=fo.types.COCODetectionDataset, + data_path=os.path.dirname(self.dataset_path), + label_types=[], # do not load the coco annotations + labels_path=self.dataset_path, + ) + self.fo_dataset.add_dynamic_sample_fields() + self.fo_dataset = self.fo_dataset.limit(self.n_samples) + + # order of coco dataset does not necessarily match the order of the fiftyone dataset + # so we create a mapping of image paths to dataset indices + # to match fiftyone samples to coco dataset samples to obtain the GT keypoints. + self.image_path_to_dataset_idx = {} + for idx, entry in enumerate(self.coco_dataset.dataset): + image_path, _ = entry + image_path = str(self.coco_dataset.dataset_dir_path / image_path) + self.image_path_to_dataset_idx[image_path] = idx + def predict_and_compute_metrics(self): with torch.no_grad(): - sample_idx = 0 - for image, keypoints in tqdm.tqdm(self.dataloader): - # [[channel1], [[[x,y],[x,y]]] + fo_sample_idx = 0 + for fo_sample in tqdm.tqdm(self.fo_dataset): + image_path = fo_sample.filepath + image_idx = self.image_path_to_dataset_idx[image_path] + image, keypoints = self.coco_dataset[image_idx] + image = image.unsqueeze(0) gt_keypoints = [] for channel in keypoints: - gt_keypoints.append([[kp[0].item(), kp[1].item()] for kp in channel]) + gt_keypoints.append([[kp[0], kp[1]] for kp in channel]) self.gt_keypoints.append(gt_keypoints) for model_name, model in self.models.items(): @@ -99,41 +117,26 @@ def predict_and_compute_metrics(self): ) for channel_idx in range(len(self.parsed_channel_config)): - self.ap_scores[model_name][sample_idx].update( + self.ap_scores[model_name][fo_sample_idx].update( {channel_idx: list(self.ap_metrics[model_name][channel_idx].compute().values())} ) - sample_idx += 1 + fo_sample_idx += 1 def visualize_predictions( self, ): """visualize keypoint detectors on a coco dataset. Requires the coco json, thechannel config and a dict of wandb checkpoints.""" - # iterate over dataset and compute predictions & gt keypoints as well as metrics - - ## create fiftyone dataset and add the predictions and gt - - fo_dataset = fo.Dataset.from_dir( - dataset_type=fo.types.COCODetectionDataset, - data_path=os.path.dirname(self.dataset_path), - label_types=[], # do not load the coco annotations - labels_path=self.dataset_path, - ) - - fo_dataset.add_dynamic_sample_fields() - - fo_dataset = fo_dataset.limit(self.n_samples) - # add the ground truth to the dataset - for sample_idx, sample in enumerate(fo_dataset): + for sample_idx, sample in enumerate(self.fo_dataset): self._add_instance_keypoints_to_fo_sample( sample, "ground_truth_keypoints", self.gt_keypoints[sample_idx], None, self.parsed_channel_config ) # add the predictions to the dataset for model_name, model in self.models.items(): - for sample_idx, sample in enumerate(fo_dataset): + for sample_idx, sample in enumerate(self.fo_dataset): keypoints, probabilities = self.predicted_keypoints[model_name][sample_idx] self._add_instance_keypoints_to_fo_sample( sample, f"{model_name}_keypoints", keypoints, probabilities, self.parsed_channel_config @@ -150,9 +153,9 @@ def visualize_predictions( # could do only one loop instead of two for the predictions usually, but we have to compute the GT keypoints, so we need to loop over the dataset anyway # https://docs.voxel51.com/user_guide/dataset_creation/index.html#model-predictions - print(fo_dataset) + print(self.fo_dataset) - session = fo.launch_app(dataset=fo_dataset) + session = fo.launch_app(dataset=self.fo_dataset) session = self._configure_session_colors(session) session.wait() @@ -230,7 +233,7 @@ def _add_instance_keypoints_to_fo_sample( "/home/tlips/Documents/synthetic-cloth-data/synthetic-cloth-data/data/datasets/TOWEL/00/annotations_val.json" ) channel_config = "corner0=corner1=corner2=corner3" - detect_only_visible_keypoints = False + detect_only_visible_keypoints = True n_samples = 50 models = {key: get_model_from_wandb_checkpoint(value) for key, value in checkpoint_dict.items()} visualizer = DetectorFiftyoneViewer(dataset_path, models, channel_config, detect_only_visible_keypoints, n_samples) From 372666aaf45a0db4655ad9c81250cf96e5268d01 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 1 Sep 2023 16:14:42 +0200 Subject: [PATCH 31/45] fix setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 98521e4..aa23a91 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ version="1.0", description="Pytorch Models, Modules etc for keypoint detection", url="https://github.com/tlpss/keypoint-detection", - packages=["keypoint_detection", "labeling"], + packages=["keypoint_detection"], install_requires=[ "torch>=0.10", "torchvision>=0.11", From fa3f86fa9fb91d9c6497ffb7415bcdc46c5b02f8 Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 12 Sep 2023 08:08:53 +0200 Subject: [PATCH 32/45] fix crop augmentations for non-square images --- keypoint_detection/data/datamodule.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/keypoint_detection/data/datamodule.py b/keypoint_detection/data/datamodule.py index 30b086c..bfd1446 100644 --- a/keypoint_detection/data/datamodule.py +++ b/keypoint_detection/data/datamodule.py @@ -84,10 +84,13 @@ def __init__( if augment_train: print("Augmenting the training dataset!") img_height, img_width = self.train_dataset[0][0].shape[1], self.train_dataset[0][0].shape[2] + aspect_ratio = img_width / img_height train_transform = MultiChannelKeypointsCompose( [ A.ColorJitter(p=0.8), - A.RandomResizedCrop(img_height, img_width, scale=(0.8, 1.0), ratio=(0.9, 1.1), p=1.0), + A.RandomResizedCrop( + img_height, img_width, scale=(0.8, 1.0), ratio=(0.9 * aspect_ratio, 1.1 * aspect_ratio), p=1.0 + ), ] ) if isinstance(self.train_dataset, COCOKeypointsDataset): From ebbc91b08f0aca5c224ec5ba7e224aab41e03979 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 15 Sep 2023 08:30:01 +0200 Subject: [PATCH 33/45] cast keypoints to ints after transforms --- keypoint_detection/data/coco_dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/keypoint_detection/data/coco_dataset.py b/keypoint_detection/data/coco_dataset.py index 0a355e0..51ec227 100644 --- a/keypoint_detection/data/coco_dataset.py +++ b/keypoint_detection/data/coco_dataset.py @@ -99,6 +99,10 @@ def __getitem__(self, index) -> Tuple[torch.Tensor, IMG_KEYPOINTS_TYPE]: keypoints = self.dataset[index][1] + if self.transform: + transformed = self.transform(image=image, keypoints=keypoints) + image, keypoints = transformed["image"], transformed["keypoints"] + # convert all keypoints to integers values. # COCO keypoints can be floats if they specify the exact location of the keypoint (e.g. from CVAT) # even though COCO format specifies zero-indexed integers (i.e. every keypoint in the [0,1]x [0.1] pixel box becomes (0,0) @@ -110,10 +114,6 @@ def __getitem__(self, index) -> Tuple[torch.Tensor, IMG_KEYPOINTS_TYPE]: [[math.floor(keypoint[0]), math.floor(keypoint[1])] for keypoint in channel_keypoints] for channel_keypoints in keypoints ] - if self.transform: - transformed = self.transform(image=image, keypoints=keypoints) - image, keypoints = transformed["image"], transformed["keypoints"] - image = self.image_to_tensor_transform(image) return image, keypoints From e47664cfce1d886556575bff8227b60f43b00a55 Mon Sep 17 00:00:00 2001 From: tlpss Date: Mon, 25 Sep 2023 08:51:34 +0200 Subject: [PATCH 34/45] fix setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index aa23a91..59f5301 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup +from setuptools import find_packages, setup setup( name="keypoint_detection", @@ -7,7 +7,7 @@ version="1.0", description="Pytorch Models, Modules etc for keypoint detection", url="https://github.com/tlpss/keypoint-detection", - packages=["keypoint_detection"], + packages=[find_packages()], install_requires=[ "torch>=0.10", "torchvision>=0.11", From eff5db76956c055cd6ecceae1199c98cb418df51 Mon Sep 17 00:00:00 2001 From: tlpss Date: Mon, 25 Sep 2023 09:04:55 +0200 Subject: [PATCH 35/45] fix package installation --- .gitignore | 1 + keypoint_detection/train/__init__.py | 0 keypoint_detection/utils/__init__.py | 0 setup.py | 2 +- 4 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 keypoint_detection/train/__init__.py create mode 100644 keypoint_detection/utils/__init__.py diff --git a/.gitignore b/.gitignore index a9f732d..36062cb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ lightning_logs** **.ckpt **/checkpoints/ +build/** diff --git a/keypoint_detection/train/__init__.py b/keypoint_detection/train/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/keypoint_detection/utils/__init__.py b/keypoint_detection/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index 59f5301..4fc4748 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ version="1.0", description="Pytorch Models, Modules etc for keypoint detection", url="https://github.com/tlpss/keypoint-detection", - packages=[find_packages()], + packages=find_packages(exclude=("test",)), install_requires=[ "torch>=0.10", "torchvision>=0.11", From ff697647520cc2591227211b287fc7c5839d6d1c Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 29 Sep 2023 11:06:16 +0200 Subject: [PATCH 36/45] create CLI and add eval task (breaking) --- README.md | 30 ++++++---- keypoint_detection/data/datamodule.py | 36 +++++++---- keypoint_detection/models/detector.py | 4 +- .../{train => tasks}/__init__.py | 0 keypoint_detection/tasks/cli.py | 28 +++++++++ keypoint_detection/tasks/eval.py | 60 +++++++++++++++++++ keypoint_detection/{train => tasks}/train.py | 29 ++++----- .../{train/utils.py => tasks/train_utils.py} | 0 keypoint_detection/utils/visualization.py | 2 +- scripts/fiftyone_viewer.py | 36 +++++++---- setup.py | 1 + test/integration_test.sh | 2 +- test/test_detector.py | 2 +- 13 files changed, 175 insertions(+), 55 deletions(-) rename keypoint_detection/{train => tasks}/__init__.py (100%) create mode 100644 keypoint_detection/tasks/cli.py create mode 100644 keypoint_detection/tasks/eval.py rename keypoint_detection/{train => tasks}/train.py (86%) rename keypoint_detection/{train/utils.py => tasks/train_utils.py} (100%) diff --git a/README.md b/README.md index 4fd7214..8d6f550 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,16 @@ TODO: add integration example. - run `wandb login` to set up your wandb account. - you are now ready to start training. + +## Training + +To train a keypoint detector, run the `keypoint-detection train` CLI with the appropriate arguments. +To create your own configuration: run `keypoint-detection train -h` to see all parameter options and their documentation. + +A starting point could be the bash script `bash test/integration_test.sh` to test on the provided test dataset, which contains 4 images. You should see the loss going down consistently until the detector has completely overfit the train set and the loss is around the entropy of the ground truth heatmaps (if you selected the default BCE loss). + +Alternatively, you can create a sweep on [wandb](https://wandb.ai) and to then start a (number of) wandb agent(s). This is very useful for running multiple configurations (hparam search, testing on multiple datasets,..) + ## Dataset This package used the [COCO format](https://cocodataset.org/#format-data) for keypoint annotation and expects a dataset with the following structure: @@ -48,17 +58,18 @@ If you want to label data, we use[CVAT](https://github.com/opencv/cvat) labeling It is best to label your data with floats that represent the subpixel location of the keypoints. This allows for more precise resizing of the images later on. The keypoint detector cast them to ints before training to obtain the pixel they belong to (it does not support sub-pixel detections). -## Training - -There are 2 ways to train the keypoint detector: - -- The first is to run the `train.py` script with the appropriate arguments. e.g. from the root folder of this repo, you can run the bash script `bash test/integration_test.sh` to test on the provided test dataset, which contains 4 images. You should see the loss going down consistently until the detector has completely overfit the train set and the loss is around the entropy of the ground truth heatmaps (if you selected the default BCE loss). +## Evaluation +TODO +`keypoint-detection eval --help` -- The second method is to create a sweep on [wandb](https://wandb.ai) and to then start a wandb agent from the correct relative location. -A minimal sweep example is given in `test/configuration.py`. The same content should be written to a yaml file according to the wandb format. The sweep can be started by running `wandb agent ` from your CLI. +## Fiftyone viewer +TODO +`scripts/fiftyone_viewer` +## Using a trained model (Inference) +During training Pytorch Lightning will have saved checkpoints. See `scripts/checkpoint_inference.py` for a simple example to run inference with a checkpoint. +For benchmarking the inference (or training), see `scripts/benchmark.py`. -To create your own configuration: run `python train.py -h` to see all parameter options and their documentation. ## Metrics @@ -81,9 +92,6 @@ We do not use OKS as in COCO for 2 reasons: 3. (you need to estimate label variance, though you could simply set k=1 and skip this part) -## Using a trained model (Inference) -During training Pytorch Lightning will have saved checkpoints. See `scripts/checkpoint_inference.py` for a simple example to run inference with a checkpoint. -For benchmarking the inference (or training), see `scripts/benchmark.py`. ## Development info - formatting and linting is done using [pre-commit](https://pre-commit.com/) diff --git a/keypoint_detection/data/datamodule.py b/keypoint_detection/data/datamodule.py index bfd1446..36400b3 100644 --- a/keypoint_detection/data/datamodule.py +++ b/keypoint_detection/data/datamodule.py @@ -31,7 +31,7 @@ def add_argparse_args(parent_parser: argparse.ArgumentParser) -> argparse.Argume "--json_validation_dataset_path", type=str, help="Absolute path to the json file that defines the validation dataset according to the COCO format. \ - If not specified, the train dataset will be split to create a validation set.", + If not specified, the train dataset will be split to create a validation set if there is one.", ) parser.add_argument( "--json_test_dataset_path", @@ -47,11 +47,11 @@ def add_argparse_args(parent_parser: argparse.ArgumentParser) -> argparse.Argume def __init__( self, - json_dataset_path: str, keypoint_channel_configuration: list[list[str]], - batch_size: int, - validation_split_ratio: float, - num_workers: int, + batch_size: int = 16, + validation_split_ratio: float = 0.25, + num_workers: int = 2, + json_dataset_path: str = None, json_validation_dataset_path: str = None, json_test_dataset_path=None, augment_train: bool = False, @@ -62,7 +62,9 @@ def __init__( self.num_workers = num_workers self.augment_train = augment_train - self.train_dataset = COCOKeypointsDataset(json_dataset_path, keypoint_channel_configuration, **kwargs) + self.train_dataset = None + if json_dataset_path: + self.train_dataset = COCOKeypointsDataset(json_dataset_path, keypoint_channel_configuration, **kwargs) self.validation_dataset = None self.test_dataset = None @@ -72,10 +74,11 @@ def __init__( json_validation_dataset_path, keypoint_channel_configuration, **kwargs ) else: - print(f"splitting the train set to create a validation set with ratio {validation_split_ratio} ") - self.train_dataset, self.validation_dataset = KeypointsDataModule._split_dataset( - self.train_dataset, validation_split_ratio - ) + if self.train_dataset is not None: + print(f"splitting the train set to create a validation set with ratio {validation_split_ratio} ") + self.train_dataset, self.validation_dataset = KeypointsDataModule._split_dataset( + self.train_dataset, validation_split_ratio + ) if json_test_dataset_path: self.test_dataset = COCOKeypointsDataset(json_test_dataset_path, keypoint_channel_configuration, **kwargs) @@ -88,9 +91,13 @@ def __init__( train_transform = MultiChannelKeypointsCompose( [ A.ColorJitter(p=0.8), + A.RandomBrightnessContrast(p=0.8), A.RandomResizedCrop( img_height, img_width, scale=(0.8, 1.0), ratio=(0.9 * aspect_ratio, 1.1 * aspect_ratio), p=1.0 ), + A.GaussianBlur(p=0.2, blur_limit=(3, 3)), + A.Sharpen(p=0.2), + A.GaussNoise(), ] ) if isinstance(self.train_dataset, COCOKeypointsDataset): @@ -116,6 +123,9 @@ def train_dataloader(self): # but PL does for us in their seeding function: # https://lightning.ai/docs/pytorch/stable/common/trainer.html#reproducibility + if self.train_dataset is None: + return None + dataloader = DataLoader( self.train_dataset, self.batch_size, @@ -132,6 +142,9 @@ def val_dataloader(self): # but PL does for us in their seeding function: # https://lightning.ai/docs/pytorch/stable/common/trainer.html#reproducibility + if self.validation_dataset is None: + return None + dataloader = DataLoader( self.validation_dataset, self.batch_size, @@ -142,6 +155,9 @@ def val_dataloader(self): return dataloader def test_dataloader(self): + + if self.test_dataset is None: + return None dataloader = DataLoader( self.test_dataset, min(4, self.batch_size), # 4 as max for better visualization in wandb. diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index 29cd20c..d557bb4 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -327,7 +327,7 @@ def validation_step(self, val_batch, batch_idx): self.update_ap_metrics(result_dict, self.ap_validation_metrics) log_images = batch_idx == 0 and self.current_epoch > 0 and self.is_ap_epoch() - if log_images: + if log_images and isinstance(self.logger, pl.loggers.wandb.WandbLogger): image_grids = self.visualize_predictions_channels(result_dict) self.log_image_grids(image_grids, mode="validation") @@ -340,7 +340,7 @@ def test_step(self, test_batch, batch_idx): result_dict = self.shared_step(test_batch, batch_idx, include_visualization_data_in_result_dict=True) self.update_ap_metrics(result_dict, self.ap_test_metrics) # only log first 10 batches to reduce storage space - if batch_idx < 10: + if batch_idx < 10 and isinstance(self.logger, pl.loggers.wandb.WandbLogger): image_grids = self.visualize_predictions_channels(result_dict) self.log_image_grids(image_grids, mode="test") self.log("test/epoch_loss", result_dict["loss"]) diff --git a/keypoint_detection/train/__init__.py b/keypoint_detection/tasks/__init__.py similarity index 100% rename from keypoint_detection/train/__init__.py rename to keypoint_detection/tasks/__init__.py diff --git a/keypoint_detection/tasks/cli.py b/keypoint_detection/tasks/cli.py new file mode 100644 index 0000000..09e7649 --- /dev/null +++ b/keypoint_detection/tasks/cli.py @@ -0,0 +1,28 @@ +"""cli entry point""" +import sys + +from keypoint_detection.tasks.eval import eval_cli +from keypoint_detection.tasks.train import train_cli + +TRAIN_TASK = "train" +EVAL_TASK = "eval" +TASKS = [TRAIN_TASK, EVAL_TASK] + + +def main(): + # read command line args in plain python + + # TODO this is a very hacky approach for combining independent cli scripts + # should redesign this in the future. + + print(sys.argv) + task = sys.argv[1] + sys.argv.pop(1) + + if task == "--help" or task == "-h": + print("Usage: keypoint-detection [task] [task args]") + print(f"Tasks: {TASKS}") + elif task == TRAIN_TASK: + train_cli() + elif task == EVAL_TASK: + eval_cli() diff --git a/keypoint_detection/tasks/eval.py b/keypoint_detection/tasks/eval.py new file mode 100644 index 0000000..7f7465e --- /dev/null +++ b/keypoint_detection/tasks/eval.py @@ -0,0 +1,60 @@ +"""run evaluation on a model for the given dataset""" + + +import pytorch_lightning as pl +import torch + +from keypoint_detection.data.datamodule import KeypointsDataModule +from keypoint_detection.models.detector import KeypointDetector +from keypoint_detection.utils.load_checkpoints import get_model_from_wandb_checkpoint + + +def evaluate_model(model: KeypointDetector, datamodule: KeypointsDataModule) -> None: + """evaluate the model on the given datamodule and checkpoint path""" + + device = "cuda" if torch.cuda.is_available() else "cpu" + + model.to(device) + model.eval() + + trainer = pl.Trainer( + gpus=1 if torch.cuda.is_available() else 0, + deterministic=True, + ) + output = trainer.test(model, datamodule) + return output + + +def eval_cli(): + argparser = argparse.ArgumentParser() + argparser.add_argument( + "--wandb_checkpoint", type=str, required=True, help="The wandb checkpoint to load the model from" + ) + argparser.add_argument( + "--test_json_path", + type=str, + required=True, + help="The path to the json file that defines the test dataset according to the COCO format.", + ) + args = argparser.parse_args() + + wandb_checkpoint = args.wandb_checkpoint + test_json_path = args.test_json_path + + model = get_model_from_wandb_checkpoint(wandb_checkpoint) + data_module = KeypointsDataModule( + model.keypoint_channel_configuration, json_test_dataset_path=test_json_path, batch_size=8 + ) + evaluate_model(model, data_module) + + +if __name__ == "__main__": + import argparse + + wandb_checkpoint = "tlips/synthetic-cloth-keypoints-single-towel/model-gl39yjtf:v0" + test_json_path = "/home/tlips/Documents/synthetic-cloth-data/synthetic-cloth-data/data/datasets/TOWEL/07-purple-towel-on-white/annotations_val.json" + model = get_model_from_wandb_checkpoint(wandb_checkpoint) + data_module = KeypointsDataModule( + model.keypoint_channel_configuration, json_test_dataset_path=test_json_path, batch_size=8 + ) + output = evaluate_model(model, data_module) diff --git a/keypoint_detection/train/train.py b/keypoint_detection/tasks/train.py similarity index 86% rename from keypoint_detection/train/train.py rename to keypoint_detection/tasks/train.py index e703a80..d99f3e3 100644 --- a/keypoint_detection/train/train.py +++ b/keypoint_detection/tasks/train.py @@ -1,3 +1,4 @@ +"""train detector based on argparse configuration""" from argparse import ArgumentParser from typing import Tuple @@ -9,7 +10,7 @@ from keypoint_detection.data.datamodule import KeypointsDataModule from keypoint_detection.models.backbones.backbone_factory import BackboneFactory from keypoint_detection.models.detector import KeypointDetector -from keypoint_detection.train.utils import create_pl_trainer, parse_channel_configuration +from keypoint_detection.tasks.train_utils import create_pl_trainer, parse_channel_configuration from keypoint_detection.utils.path import get_wandb_log_dir_path @@ -52,7 +53,7 @@ def add_system_args(parent_parser: ArgumentParser) -> ArgumentParser: return parent_parser -def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: +def train(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: """ Initializes the datamodule, model and trainer based on the global hyperparameters. calls trainer.fit(model, module) afterwards and returns both model and trainer. @@ -61,21 +62,7 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: pl.seed_everything(hparams["seed"], workers=True) # use deterministic algorithms for torch to ensure exact reproducibility - # https://pytorch.org/docs/stable/notes/randomness.html#reproducibility - # this can slow down training - # but the impact is limited in my experience. - # so I prefer to be deterministic (and hence reproducible) by default. - - # also note that following is not enough: - # torch.backends.cudnn.deterministic = True - # there are other non-deterministic algorithms - # cf list at https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html#torch.use_deterministic_algorithms - - # the following is still not good enough with Pytorch-Lightning: - # import torch - # torch.use_deterministic_algorithms(True) - # though I am not exactly sure why. - # so we have to set it in the trainer! (see create_pl_trainer) + # we have to set it in the trainer! (see create_pl_trainer) backbone = BackboneFactory.create_backbone(**hparams) model = KeypointDetector(backbone=backbone, **hparams) @@ -105,7 +92,7 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: return model, trainer -if __name__ == "__main__": +def train_cli(): """ 1. creates argumentparser with Model, Trainer and system paramaters; which can be used to overwrite default parameters when running python train.py -- @@ -145,4 +132,8 @@ def main(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: print(f" config after wandb init: {hparams}") print("starting training") - main(hparams) + train(hparams) + + +if __name__ == "__main__": + train_cli() diff --git a/keypoint_detection/train/utils.py b/keypoint_detection/tasks/train_utils.py similarity index 100% rename from keypoint_detection/train/utils.py rename to keypoint_detection/tasks/train_utils.py diff --git a/keypoint_detection/utils/visualization.py b/keypoint_detection/utils/visualization.py index d332f38..ea17a1d 100644 --- a/keypoint_detection/utils/visualization.py +++ b/keypoint_detection/utils/visualization.py @@ -70,7 +70,7 @@ def visualize_predicted_heatmaps( from torch.utils.data import DataLoader from keypoint_detection.data.coco_dataset import COCOKeypointsDataset - from keypoint_detection.train.train import parse_channel_configuration + from keypoint_detection.tasks.train import parse_channel_configuration from keypoint_detection.utils.heatmap import create_heatmap_batch parser = ArgumentParser() diff --git a/scripts/fiftyone_viewer.py b/scripts/fiftyone_viewer.py index bfc7660..0830c47 100644 --- a/scripts/fiftyone_viewer.py +++ b/scripts/fiftyone_viewer.py @@ -11,10 +11,13 @@ from keypoint_detection.data.coco_dataset import COCOKeypointsDataset from keypoint_detection.models.detector import KeypointDetector from keypoint_detection.models.metrics import DetectedKeypoint, Keypoint, KeypointAPMetrics -from keypoint_detection.train.utils import parse_channel_configuration +from keypoint_detection.tasks.train_utils import parse_channel_configuration from keypoint_detection.utils.heatmap import compute_keypoint_probability, get_keypoints_from_heatmap_batch_maxpool from keypoint_detection.utils.load_checkpoints import get_model_from_wandb_checkpoint +# TODO: can get channel config from the models! no need to specify manually +# TODO: mAP / image != mAP, maybe it is also not even the best metric to use for ordering samples .Should also log the loss / image. + class DetectorFiftyoneViewer: def __init__( @@ -155,7 +158,7 @@ def visualize_predictions( print(self.fo_dataset) - session = fo.launch_app(dataset=self.fo_dataset) + session = fo.launch_app(dataset=self.fo_dataset, port=5252) session = self._configure_session_colors(session) session.wait() @@ -220,22 +223,35 @@ def _add_instance_keypoints_to_fo_sample( return sample +import cv2 + +cv2.INTER_LINEAR if __name__ == "__main__": # TODO: make CLI for this -> hydra config? checkpoint_dict = { # "maxvit-256-flat": "tlips/synthetic-cloth-keypoints-quest-for-precision/model-5ogj44k0:v0", # "maxvit-512-flat": "tlips/synthetic-cloth-keypoints-quest-for-precision/model-1of5e6qs:v0", - "maxvit-pyflex-20k": "tlips/synthetic-cloth-keypoints/model-qiellxgb:v0" + # "maxvit-pyflex-20k": "tlips/synthetic-cloth-keypoints/model-qiellxgb:v0" + # "maxvit-pyflex-512x256": "tlips/synthetic-cloth-keypoints/model-8m3z0wyo:v0", + # "maxvit-RTF-512x256" : "tlips/synthetic-cloth-keypoints/model-pzbwimqa:v0", + # "maxvit-sim-longer": "tlips/synthetic-cloth-keypoints/model-nvs1pktv:v0", + # "rtf-cv2":"tlips/synthetic-cloth-keypoints/model-xvkowjqr:v0", + # "rtf-pil":"tlips/synthetic-cloth-keypoints/model-0goi5hc7:v0", + # "sim-new-data":"tlips/synthetic-cloth-keypoints/model-axrqhql1:v0", + # "sim-40k":"tlips/synthetic-cloth-keypoints/model-yillsdva:v0" + # "purple-towel-on-white": "tlips/synthetic-cloth-keypoints-single-towel/model-pw2tsued:v0", + "purple-towel-on-white-separate": "tlips/synthetic-cloth-keypoints-single-towel/model-gl39yjtf:v0" } - dataset_path = "/storage/users/tlips/RTFClothes/towels-test_resized_256x256/towels-test.json" - dataset_path = ( - "/home/tlips/Documents/synthetic-cloth-data/synthetic-cloth-data/data/datasets/TOWEL/00/annotations_val.json" - ) - channel_config = "corner0=corner1=corner2=corner3" + dataset_path = "/storage/users/tlips/aRTFClothes/towels-test_resized_512x256/towels-test.json" + dataset_path = "/home/tlips/Documents/synthetic-cloth-data/synthetic-cloth-data/data/datasets/TOWEL/05-512x256-40k/annotations_val.json" + dataset_path = "/home/tlips/Documents/synthetic-cloth-data/synthetic-cloth-data/data/datasets/TOWEL/07-purple-towel-on-white/annotations_val.json" + channel_config = "corner0;corner1;corner2;corner3" detect_only_visible_keypoints = True - n_samples = 50 + n_samples = 200 models = {key: get_model_from_wandb_checkpoint(value) for key, value in checkpoint_dict.items()} - visualizer = DetectorFiftyoneViewer(dataset_path, models, channel_config, detect_only_visible_keypoints, n_samples) + visualizer = DetectorFiftyoneViewer( + dataset_path, models, channel_config, detect_only_visible_keypoints, n_samples, ap_threshold_distances=[4] + ) visualizer.predict_and_compute_metrics() visualizer.visualize_predictions() diff --git a/setup.py b/setup.py index 4fc4748..d6917ba 100644 --- a/setup.py +++ b/setup.py @@ -26,4 +26,5 @@ "pydantic", "fiftyone", ], + entry_points={"console_scripts": ["keypoint-detection = keypoint_detection.tasks.cli:main"]}, ) diff --git a/test/integration_test.sh b/test/integration_test.sh index 96b9380..20203be 100644 --- a/test/integration_test.sh +++ b/test/integration_test.sh @@ -3,7 +3,7 @@ # Run from the repo's root folder using bash test/integration_test.sh # make sure to remove all trailing spaces from the command, as this would result in an error when using bash. -python keypoint_detection/train/train.py \ +python keypoint_detection/tasks/train.py \ --keypoint_channel_configuration "box_corner0= box_corner1 = box_corner2= box_corner3; flap_corner0 ; flap_corner2" \ --json_dataset_path "test/test_dataset/coco_dataset.json" --json_validation_dataset_path "test/test_dataset/coco_dataset.json" --batch_size 2 --wandb_project "keypoint-detector-integration-test" \ --max_epochs 50 --early_stopping_relative_threshold -1.0 --log_every_n_steps 1 --accelerator="gpu" --devices 1 --precision 16 --augment_train diff --git a/test/test_detector.py b/test/test_detector.py index 5bde578..a09c5b4 100644 --- a/test/test_detector.py +++ b/test/test_detector.py @@ -10,7 +10,7 @@ from keypoint_detection.models.backbones.unet import Unet from keypoint_detection.models.detector import KeypointDetector from keypoint_detection.models.metrics import KeypointAPMetric -from keypoint_detection.train.utils import create_pl_trainer +from keypoint_detection.tasks.train_utils import create_pl_trainer from keypoint_detection.utils.heatmap import create_heatmap_batch, generate_channel_heatmap from keypoint_detection.utils.load_checkpoints import load_from_checkpoint from keypoint_detection.utils.path import get_wandb_log_dir_path From 58f59379ed5c6cdf5c90cc5cd7051cb67fa387ab Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 29 Sep 2023 13:56:00 +0200 Subject: [PATCH 37/45] change channel delimiter to ':' because ';' is used by bash --- keypoint_detection/tasks/train_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keypoint_detection/tasks/train_utils.py b/keypoint_detection/tasks/train_utils.py index f909de6..94d8027 100644 --- a/keypoint_detection/tasks/train_utils.py +++ b/keypoint_detection/tasks/train_utils.py @@ -97,6 +97,6 @@ def create_pl_trainer(hparams: dict, wandb_logger: WandbLogger) -> Trainer: def parse_channel_configuration(channel_configuration: str) -> List[List[str]]: assert isinstance(channel_configuration, str) - channels = channel_configuration.split(";") + channels = channel_configuration.split(":") channels = [[category.strip() for category in channel.split("=")] for channel in channels] return channels From 64b3bdd94fdf6d7b348c6cef72c1e77bed6ebe57 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 29 Sep 2023 13:57:28 +0200 Subject: [PATCH 38/45] change delimiter --- keypoint_detection/tasks/train.py | 2 +- test/integration_test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/keypoint_detection/tasks/train.py b/keypoint_detection/tasks/train.py index d99f3e3..bd4f6e3 100644 --- a/keypoint_detection/tasks/train.py +++ b/keypoint_detection/tasks/train.py @@ -29,7 +29,7 @@ def add_system_args(parent_parser: ArgumentParser) -> ArgumentParser: parser.add_argument( "--keypoint_channel_configuration", type=str, - help="A list of the semantic keypoints that you want to learn in each channel. These semantic categories must be defined in the COCO dataset. Seperate the channels with a ; and the categories within a channel with a =", + help="A list of the semantic keypoints that you want to learn in each channel. These semantic categories must be defined in the COCO dataset. Seperate the channels with a : and the categories within a channel with a =", ) parser.add_argument( diff --git a/test/integration_test.sh b/test/integration_test.sh index 20203be..537c9d5 100644 --- a/test/integration_test.sh +++ b/test/integration_test.sh @@ -4,6 +4,6 @@ # make sure to remove all trailing spaces from the command, as this would result in an error when using bash. python keypoint_detection/tasks/train.py \ ---keypoint_channel_configuration "box_corner0= box_corner1 = box_corner2= box_corner3; flap_corner0 ; flap_corner2" \ +--keypoint_channel_configuration "box_corner0= box_corner1 = box_corner2= box_corner3: flap_corner0:flap_corner2" \ --json_dataset_path "test/test_dataset/coco_dataset.json" --json_validation_dataset_path "test/test_dataset/coco_dataset.json" --batch_size 2 --wandb_project "keypoint-detector-integration-test" \ --max_epochs 50 --early_stopping_relative_threshold -1.0 --log_every_n_steps 1 --accelerator="gpu" --devices 1 --precision 16 --augment_train From d0cbb053aed347f63b5ccb841f7ecd003f0c81b7 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 29 Sep 2023 13:57:45 +0200 Subject: [PATCH 39/45] safe division --- keypoint_detection/models/metrics.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/keypoint_detection/models/metrics.py b/keypoint_detection/models/metrics.py index cdda141..1b4905f 100644 --- a/keypoint_detection/models/metrics.py +++ b/keypoint_detection/models/metrics.py @@ -132,8 +132,8 @@ def calculate_precision_recall( else: false_positives += 1 - precision.append(true_positives / (true_positives + false_positives)) - recall.append(true_positives / total_ground_truth_keypoints) + precision.append(_zero_aware_division(true_positives, (true_positives + false_positives))) + recall.append(_zero_aware_division(true_positives, total_ground_truth_keypoints)) precision.append(0.0) recall.append(1.0) @@ -230,6 +230,15 @@ def reset(self) -> None: metric.reset() +def _zero_aware_division(num: float, denom: float) -> float: + if num == 0: + return 0 + if denom == 0 and num != 0: + return float("inf") + else: + return num / denom + + if __name__ == "__main__": print( check_forward_full_state_property( From 9ffa9a2f70936fb1ae82a2fffd96f7a6e3092f06 Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 10 Oct 2023 12:53:39 +0200 Subject: [PATCH 40/45] visualize all keypoints on single image --- keypoint_detection/models/detector.py | 56 ++++++++----- keypoint_detection/tasks/inference.py | 41 ++++++++++ keypoint_detection/utils/visualization.py | 97 ++++++++++++++++++++--- 3 files changed, 160 insertions(+), 34 deletions(-) create mode 100644 keypoint_detection/tasks/inference.py diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index d557bb4..6994bdb 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -10,7 +10,11 @@ from keypoint_detection.models.backbones.base_backbone import Backbone from keypoint_detection.models.metrics import DetectedKeypoint, Keypoint, KeypointAPMetrics from keypoint_detection.utils.heatmap import BCE_loss, create_heatmap_batch, get_keypoints_from_heatmap_batch_maxpool -from keypoint_detection.utils.visualization import visualize_predicted_heatmaps +from keypoint_detection.utils.visualization import ( + get_logging_label_from_channel_configuration, + visualize_predicted_heatmaps, + visualize_predicted_keypoints, +) class KeypointDetector(pl.LightningModule): @@ -266,7 +270,7 @@ def training_step(self, train_batch, batch_idx): if log_images: image_grids = self.visualize_predictions_channels(result_dict) - self.log_image_grids(image_grids, mode="train") + self.log_channel_predictions_grids(image_grids, mode="train") for channel_name in self.keypoint_channel_configuration: self.log(f"train/{channel_name}", result_dict[f"{channel_name}_loss"]) @@ -299,26 +303,29 @@ def visualize_predictions_channels(self, result_dict): image_grids.append(grid) return image_grids - @staticmethod - def logging_label(channel_configuration, mode: str) -> str: - channel_name = channel_configuration - - if isinstance(channel_configuration, list): - if len(channel_configuration) == 1: - channel_name = channel_configuration[0] - else: - channel_name = f"{channel_configuration[0]}+{channel_configuration[1]}+..." - - channel_name_short = (channel_name[:40] + "...") if len(channel_name) > 40 else channel_name - label = f"{channel_name_short}_{mode}_keypoints" - return label - - def log_image_grids(self, image_grids, mode: str): + def log_channel_predictions_grids(self, image_grids, mode: str): for channel_configuration, grid in zip(self.keypoint_channel_configuration, image_grids): - label = KeypointDetector.logging_label(channel_configuration, mode) + label = get_logging_label_from_channel_configuration(channel_configuration, mode) image_caption = "top: predicted heatmaps, bottom: gt heatmaps" self.logger.experiment.log({label: wandb.Image(grid, caption=image_caption)}) + def visualize_predicted_keypoints(self, result_dict): + images = result_dict["input_images"] + predicted_heatmaps = result_dict["predicted_heatmaps"] + # get the keypoints from the heatmaps + predicted_heatmaps = predicted_heatmaps.detach().float() + predicted_keypoints = get_keypoints_from_heatmap_batch_maxpool( + predicted_heatmaps, self.max_keypoints, self.minimal_keypoint_pixel_distance, abs_max_threshold=0.4 + ) + # overlay the images with the keypoints + grid = visualize_predicted_keypoints(images, predicted_keypoints, self.keypoint_channel_configuration) + return grid + + def log_predicted_keypoints(self, grid, mode=str): + label = f"predicted_keypoints_{mode}" + image_caption = "predicted keypoints" + self.logger.experiment.log({label: wandb.Image(grid, caption=image_caption)}) + def validation_step(self, val_batch, batch_idx): # no need to switch model to eval mode, this is handled by pytorch lightning result_dict = self.shared_step(val_batch, batch_idx, include_visualization_data_in_result_dict=True) @@ -328,8 +335,11 @@ def validation_step(self, val_batch, batch_idx): log_images = batch_idx == 0 and self.current_epoch > 0 and self.is_ap_epoch() if log_images and isinstance(self.logger, pl.loggers.wandb.WandbLogger): - image_grids = self.visualize_predictions_channels(result_dict) - self.log_image_grids(image_grids, mode="validation") + channel_grids = self.visualize_predictions_channels(result_dict) + self.log_channel_predictions_grids(channel_grids, mode="validation") + + keypoint_grids = self.visualize_predicted_keypoints(result_dict) + self.log_predicted_keypoints(keypoint_grids, mode="validation") ## log (defaults to on_epoch, which aggregates the logged values over entire validation set) self.log("validation/epoch_loss", result_dict["loss"]) @@ -342,7 +352,11 @@ def test_step(self, test_batch, batch_idx): # only log first 10 batches to reduce storage space if batch_idx < 10 and isinstance(self.logger, pl.loggers.wandb.WandbLogger): image_grids = self.visualize_predictions_channels(result_dict) - self.log_image_grids(image_grids, mode="test") + self.log_channel_predictions_grids(image_grids, mode="test") + + keypoint_grids = self.visualize_predicted_keypoints(result_dict) + self.log_predicted_keypoints(keypoint_grids, mode="validation") + self.log("test/epoch_loss", result_dict["loss"]) self.log("test/gt_loss", result_dict["gt_loss"]) diff --git a/keypoint_detection/tasks/inference.py b/keypoint_detection/tasks/inference.py new file mode 100644 index 0000000..b6bae63 --- /dev/null +++ b/keypoint_detection/tasks/inference.py @@ -0,0 +1,41 @@ +""" run inference on a provided image and save the result to a file """ + +import numpy as np +import torch +from PIL import Image + +from keypoint_detection.models.detector import KeypointDetector +from keypoint_detection.utils.heatmap import get_keypoints_from_heatmap_batch_maxpool +from keypoint_detection.utils.load_checkpoints import get_model_from_wandb_checkpoint +from keypoint_detection.utils.visualization import draw_keypoints_on_image + + +def run_inference(model: KeypointDetector, image, confidence_threshold: float = 0.1) -> Image: + model.eval() + tensored_image = torch.from_numpy(np.array(image)).float() + tensored_image = tensored_image / 255.0 + tensored_image = tensored_image.permute(2, 0, 1) + tensored_image = tensored_image.unsqueeze(0) + with torch.no_grad(): + heatmaps = model(tensored_image) + + keypoints = get_keypoints_from_heatmap_batch_maxpool(heatmaps, abs_max_threshold=confidence_threshold) + image_keypoints = keypoints[0] + for keypoints, channel_config in zip(image_keypoints, model.keypoint_channel_configuration): + print(f"Keypoints for {channel_config}: {keypoints}") + image = draw_keypoints_on_image(image, image_keypoints, model.keypoint_channel_configuration) + return image + + +if __name__ == "__main__": + wandb_checkpoint = "tlips/synthetic-lego-battery-keypoints/model-tbzd50z8:v0" + image_path = "/home/tlips/Downloads/Lego-battery-real/0.jpg" + # image_path = "/home/tlips/Documents/synthetic-cloth-data/synthetic-cloth-data/data/datasets/LEGO-battery/01/images/0.jpg" + image_size = (256, 256) + + image = Image.open(image_path) + image = image.resize(image_size) + + model = get_model_from_wandb_checkpoint(wandb_checkpoint) + image = run_inference(model, image) + image.save("inference_result.png") diff --git a/keypoint_detection/utils/visualization.py b/keypoint_detection/utils/visualization.py index ea17a1d..0bca3af 100644 --- a/keypoint_detection/utils/visualization.py +++ b/keypoint_detection/utils/visualization.py @@ -1,13 +1,29 @@ from argparse import ArgumentParser -from typing import List +from typing import List, Tuple +import numpy as np import torch import torchvision from matplotlib import cm +from PIL import Image, ImageDraw, ImageFont from keypoint_detection.utils.heatmap import generate_channel_heatmap +def get_logging_label_from_channel_configuration(channel_configuration: List[List[str]], mode: str) -> str: + channel_name = channel_configuration + + if isinstance(channel_configuration, list): + if len(channel_configuration) == 1: + channel_name = channel_configuration[0] + else: + channel_name = f"{channel_configuration[0]}+{channel_configuration[1]}+..." + + channel_name_short = (channel_name[:40] + "...") if len(channel_name) > 40 else channel_name + label = f"{channel_name_short}_{mode}" + return label + + def overlay_image_with_heatmap(images: torch.Tensor, heatmaps: torch.Tensor, alpha=0.5) -> torch.Tensor: """ """ viridis = cm.get_cmap("viridis") @@ -19,7 +35,22 @@ def overlay_image_with_heatmap(images: torch.Tensor, heatmaps: torch.Tensor, alp return overlayed_images -def overlay_image_with_keypoints(images: torch.Tensor, keypoints: List[torch.Tensor], sigma: float) -> torch.Tensor: +def visualize_predicted_heatmaps( + imgs: torch.Tensor, + predicted_heatmaps: torch.Tensor, + gt_heatmaps: torch.Tensor, +): + num_images = min(predicted_heatmaps.shape[0], 6) + + predicted_heatmap_overlays = overlay_image_with_heatmap(imgs[:num_images], predicted_heatmaps[:num_images]) + gt_heatmap_overlays = overlay_image_with_heatmap(imgs[:num_images], gt_heatmaps[:num_images]) + + images = torch.cat([predicted_heatmap_overlays, gt_heatmap_overlays]) + grid = torchvision.utils.make_grid(images, nrow=num_images) + return grid + + +def overlay_images_with_keypoints(images: torch.Tensor, keypoints: List[torch.Tensor], sigma: float) -> torch.Tensor: """ images N x 3 x H x W keypoints list of size N with Tensors C x 2 @@ -49,18 +80,58 @@ def overlay_image_with_keypoints(images: torch.Tensor, keypoints: List[torch.Ten return overlayed_images -def visualize_predicted_heatmaps( - imgs: torch.Tensor, - predicted_heatmaps: torch.Tensor, - gt_heatmaps: torch.Tensor, +def draw_keypoints_on_image( + image: Image, image_keypoints: List[List[Tuple[int, int]]], channel_configuration: List[List[str]] +) -> Image: + """adds all keypoints to the PIL image, with different colors for each channel.""" + color_pool = [ + "#FF00FF", # Neon Purple + "#00FF00", # Electric Green + "#FFFF00", # Cyber Yellow + "#0000FF", # Laser Blue + "#FF0000", # Radioactive Red + "#00FFFF", # Galactic Teal + "#FF00AA", # Quantum Pink + "#C0C0C0", # Holographic Silver + "#000000", # Abyssal Black + "#FFA500", # Cosmic Orange + ] + image_size = image.size + min_size = min(image_size) + scale = 1 + (min_size // 256) + + draw = ImageDraw.Draw(image) + for channel_idx, channel_keypoints in enumerate(image_keypoints): + for keypoint_idx, keypoint in enumerate(channel_keypoints): + u, v = keypoint + draw.ellipse((u - scale, v - scale, u + scale, v + scale), fill=color_pool[channel_idx]) + + draw.text( + (10, channel_idx * 10 * scale), + get_logging_label_from_channel_configuration(channel_configuration[channel_idx], "").split("_")[0], + fill=color_pool[channel_idx], + font=ImageFont.truetype("FreeMono.ttf", size=10 * scale), + ) + + return image + + +def visualize_predicted_keypoints( + images: torch.Tensor, keypoints: List[List[List[List[int]]]], channel_configuration: List[List[str]] ): - num_images = min(predicted_heatmaps.shape[0], 6) - - predicted_heatmap_overlays = overlay_image_with_heatmap(imgs[:num_images], predicted_heatmaps[:num_images]) - gt_heatmap_overlays = overlay_image_with_heatmap(imgs[:num_images], gt_heatmaps[:num_images]) - - images = torch.cat([predicted_heatmap_overlays, gt_heatmap_overlays]) - grid = torchvision.utils.make_grid(images, nrow=num_images) + drawn_images = [] + num_images = min(images.shape[0], 6) + for i in range(num_images): + # PIL expects uint8 images + image = images[i].permute(1, 2, 0).numpy() * 255 + image = image.astype(np.uint8) + image = Image.fromarray(image) + image = draw_keypoints_on_image(image, keypoints[i], channel_configuration) + drawn_images.append(image) + + drawn_images = torch.stack([torch.from_numpy(np.array(image)).permute(2, 0, 1) / 255 for image in drawn_images]) + + grid = torchvision.utils.make_grid(drawn_images, nrow=num_images) return grid From 2acddc5d108e07260b8856d9c789aa5513a73e71 Mon Sep 17 00:00:00 2001 From: tlpss Date: Tue, 10 Oct 2023 15:03:29 +0200 Subject: [PATCH 41/45] bugfix --- keypoint_detection/models/detector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index 6994bdb..8e3ed59 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -315,7 +315,7 @@ def visualize_predicted_keypoints(self, result_dict): # get the keypoints from the heatmaps predicted_heatmaps = predicted_heatmaps.detach().float() predicted_keypoints = get_keypoints_from_heatmap_batch_maxpool( - predicted_heatmaps, self.max_keypoints, self.minimal_keypoint_pixel_distance, abs_max_threshold=0.4 + predicted_heatmaps, self.max_keypoints, self.minimal_keypoint_pixel_distance, abs_max_threshold=0.2 ) # overlay the images with the keypoints grid = visualize_predicted_keypoints(images, predicted_keypoints, self.keypoint_channel_configuration) @@ -355,7 +355,7 @@ def test_step(self, test_batch, batch_idx): self.log_channel_predictions_grids(image_grids, mode="test") keypoint_grids = self.visualize_predicted_keypoints(result_dict) - self.log_predicted_keypoints(keypoint_grids, mode="validation") + self.log_predicted_keypoints(keypoint_grids, mode="test") self.log("test/epoch_loss", result_dict["loss"]) self.log("test/gt_loss", result_dict["gt_loss"]) From 6996fe49c5563e36f6a871c46f3a5a22916e2cc3 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 20 Oct 2023 09:41:50 +0200 Subject: [PATCH 42/45] reduce threshold and fix bug for visualisation --- keypoint_detection/models/detector.py | 2 +- keypoint_detection/utils/visualization.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/keypoint_detection/models/detector.py b/keypoint_detection/models/detector.py index 8e3ed59..f62b62a 100644 --- a/keypoint_detection/models/detector.py +++ b/keypoint_detection/models/detector.py @@ -315,7 +315,7 @@ def visualize_predicted_keypoints(self, result_dict): # get the keypoints from the heatmaps predicted_heatmaps = predicted_heatmaps.detach().float() predicted_keypoints = get_keypoints_from_heatmap_batch_maxpool( - predicted_heatmaps, self.max_keypoints, self.minimal_keypoint_pixel_distance, abs_max_threshold=0.2 + predicted_heatmaps, self.max_keypoints, self.minimal_keypoint_pixel_distance, abs_max_threshold=0.1 ) # overlay the images with the keypoints grid = visualize_predicted_keypoints(images, predicted_keypoints, self.keypoint_channel_configuration) diff --git a/keypoint_detection/utils/visualization.py b/keypoint_detection/utils/visualization.py index 0bca3af..b547513 100644 --- a/keypoint_detection/utils/visualization.py +++ b/keypoint_detection/utils/visualization.py @@ -20,7 +20,10 @@ def get_logging_label_from_channel_configuration(channel_configuration: List[Lis channel_name = f"{channel_configuration[0]}+{channel_configuration[1]}+..." channel_name_short = (channel_name[:40] + "...") if len(channel_name) > 40 else channel_name - label = f"{channel_name_short}_{mode}" + if mode != "": + label = f"{channel_name_short}_{mode}" + else: + label = channel_name_short return label @@ -108,7 +111,7 @@ def draw_keypoints_on_image( draw.text( (10, channel_idx * 10 * scale), - get_logging_label_from_channel_configuration(channel_configuration[channel_idx], "").split("_")[0], + get_logging_label_from_channel_configuration(channel_configuration[channel_idx], ""), fill=color_pool[channel_idx], font=ImageFont.truetype("FreeMono.ttf", size=10 * scale), ) From 2822dc55ce28ad21c9bb3940c01c71a765430c44 Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 20 Oct 2023 09:49:10 +0200 Subject: [PATCH 43/45] fix missing import --- keypoint_detection/tasks/eval.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/keypoint_detection/tasks/eval.py b/keypoint_detection/tasks/eval.py index 7f7465e..bc4cb7c 100644 --- a/keypoint_detection/tasks/eval.py +++ b/keypoint_detection/tasks/eval.py @@ -1,6 +1,8 @@ """run evaluation on a model for the given dataset""" +import argparse + import pytorch_lightning as pl import torch @@ -49,10 +51,10 @@ def eval_cli(): if __name__ == "__main__": - import argparse wandb_checkpoint = "tlips/synthetic-cloth-keypoints-single-towel/model-gl39yjtf:v0" test_json_path = "/home/tlips/Documents/synthetic-cloth-data/synthetic-cloth-data/data/datasets/TOWEL/07-purple-towel-on-white/annotations_val.json" + test_json_path = "/storage/users/tlips/aRTFClothes/cloth-on-white/purple-towel-on-white_resized_512x256/purple-towel-on-white.json" model = get_model_from_wandb_checkpoint(wandb_checkpoint) data_module = KeypointsDataModule( model.keypoint_channel_configuration, json_test_dataset_path=test_json_path, batch_size=8 From 66e9d5e20dbf75c87055b360e06e84b87e26316c Mon Sep 17 00:00:00 2001 From: tlpss Date: Fri, 20 Oct 2023 11:52:22 +0200 Subject: [PATCH 44/45] option to start training from checkpoint --- README.md | 12 ++++++++---- keypoint_detection/tasks/train.py | 20 ++++++++++++++++++-- keypoint_detection/utils/load_checkpoints.py | 12 ++++++++++-- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8d6f550..883dd4a 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,14 @@ TODO: add integration example. To train a keypoint detector, run the `keypoint-detection train` CLI with the appropriate arguments. To create your own configuration: run `keypoint-detection train -h` to see all parameter options and their documentation. -A starting point could be the bash script `bash test/integration_test.sh` to test on the provided test dataset, which contains 4 images. You should see the loss going down consistently until the detector has completely overfit the train set and the loss is around the entropy of the ground truth heatmaps (if you selected the default BCE loss). +A good starting point could be the bash script `bash test/integration_test.sh` to test on the provided test dataset, which contains 4 images. You should see the loss going down consistently until the detector has completely overfit the train set and the loss is around the entropy of the ground truth heatmaps (if you selected the default BCE loss). +### Wandb sweeps Alternatively, you can create a sweep on [wandb](https://wandb.ai) and to then start a (number of) wandb agent(s). This is very useful for running multiple configurations (hparam search, testing on multiple datasets,..) +### Loading pretrained weights +If you want to load pretrained keypoint detector weights, you can specify the wandb artifact of the checkpoint in the training parameters: `keypoint-detection train ..... -wandb_checkpoint_artifact `. This can be used for example to finetune on real data after pretraining on synthetic data. + ## Dataset This package used the [COCO format](https://cocodataset.org/#format-data) for keypoint annotation and expects a dataset with the following structure: @@ -48,7 +52,7 @@ This package used the [COCO format](https://cocodataset.org/#format-data) for ke dataset/ images/ ... - .json : a COCO-formatted keypoint annotation file. + .json : a COCO-formatted keypoint annotation file with filepaths relative to its parent directory. ``` For an example, see the `test_dataset` at `test/test_dataset`. @@ -66,7 +70,7 @@ TODO TODO `scripts/fiftyone_viewer` -## Using a trained model (Inference) +## Using a trained model for Inference During training Pytorch Lightning will have saved checkpoints. See `scripts/checkpoint_inference.py` for a simple example to run inference with a checkpoint. For benchmarking the inference (or training), see `scripts/benchmark.py`. @@ -86,7 +90,7 @@ In general a lower threshold will result in a lower metric. The size of this gap #TODO: add a figure to illustrate this. -We do not use OKS as in COCO for 2 reasons: +We do not use OKS as in COCO for the following reasons: 1. it requires bbox annotations, which are not always required for keypoint detection itself and represent additional label effort. 2. More importantly, in robotics the size of an object does not always correlate with the required precision. If a large and a small mug stand on a table, they require the same precise localisation of keypoints for a robot to grasp them even though their apparent size is different. 3. (you need to estimate label variance, though you could simply set k=1 and skip this part) diff --git a/keypoint_detection/tasks/train.py b/keypoint_detection/tasks/train.py index bd4f6e3..61cd51a 100644 --- a/keypoint_detection/tasks/train.py +++ b/keypoint_detection/tasks/train.py @@ -11,6 +11,7 @@ from keypoint_detection.models.backbones.backbone_factory import BackboneFactory from keypoint_detection.models.detector import KeypointDetector from keypoint_detection.tasks.train_utils import create_pl_trainer, parse_channel_configuration +from keypoint_detection.utils.load_checkpoints import get_model_from_wandb_checkpoint from keypoint_detection.utils.path import get_wandb_log_dir_path @@ -49,6 +50,12 @@ def add_system_args(parent_parser: ArgumentParser) -> ArgumentParser: help="do not use deterministic algorithms for pytorch. This can speed up training, but will make it non-reproducible.", ) + parser.add_argument( + "--wandb_checkpoint_artifact", + type=str, + help="A checkpoint to resume/start training from. keep in mind that you currently cannot specify hyperparameters other than the LR.", + required=False, + ) parser.set_defaults(deterministic=True) return parent_parser @@ -63,9 +70,18 @@ def train(hparams: dict) -> Tuple[KeypointDetector, pl.Trainer]: # use deterministic algorithms for torch to ensure exact reproducibility # we have to set it in the trainer! (see create_pl_trainer) + if "wandb_checkpoint_artifact" in hparams.keys(): + print("Loading checkpoint from wandb") + # This will create a KeypointDetector model with the associated hyperparameters. + # Model weights will be loaded. + # Optimizer and LR scheduler will be initiated from scratch" (if you want to really resume training, you have to pass the ckeckpoint to the trainer) + # cf. https://lightning.ai/docs/pytorch/latest/common/checkpointing_basic.html#lightningmodule-from-checkpoint + model = get_model_from_wandb_checkpoint(hparams["wandb_checkpoint_artifact"]) + # TODO: how can specific hparams be overwritten here? e.g. LR reduction for finetuning or something? + else: + backbone = BackboneFactory.create_backbone(**hparams) + model = KeypointDetector(backbone=backbone, **hparams) - backbone = BackboneFactory.create_backbone(**hparams) - model = KeypointDetector(backbone=backbone, **hparams) data_module = KeypointsDataModule(**hparams) wandb_logger = WandbLogger( project=hparams["wandb_project"], diff --git a/keypoint_detection/utils/load_checkpoints.py b/keypoint_detection/utils/load_checkpoints.py index 51e7473..d43feb9 100644 --- a/keypoint_detection/utils/load_checkpoints.py +++ b/keypoint_detection/utils/load_checkpoints.py @@ -15,14 +15,17 @@ def get_model_from_wandb_checkpoint(checkpoint_reference: str): import wandb # download checkpoint locally (if not already cached) - run = wandb.init(project="inference") + if wandb.run is None: + run = wandb.init(project="inference") + else: + run = wandb.run artifact = run.use_artifact(checkpoint_reference, type="model") artifact_dir = artifact.download() checkpoint_path = Path(artifact_dir) / "model.ckpt" return load_from_checkpoint(checkpoint_path) -def load_from_checkpoint(checkpoint_path: str): +def load_from_checkpoint(checkpoint_path: str, hparams_to_override: dict = None): """ function to load a Keypoint Detector model from a local pytorch lightning checkpoint. @@ -43,3 +46,8 @@ def load_from_checkpoint(checkpoint_path: str): backbone = BackboneFactory.create_backbone(**checkpoint["hyper_parameters"]) model = KeypointDetector.load_from_checkpoint(checkpoint_path, backbone=backbone) return model + + +if __name__ == "__main__": + model = get_model_from_wandb_checkpoint("tlips/synthetic-cloth-keypoints-tshirts/model-4um302zo:v0") + print(model.hparams) From 15648f74c39a957322e0cf4ce284a018c8bd777a Mon Sep 17 00:00:00 2001 From: tlpss Date: Wed, 25 Oct 2023 13:37:12 +0200 Subject: [PATCH 45/45] exclude checkpoint test on gh actions for now --- test/test_detector.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_detector.py b/test/test_detector.py index a09c5b4..8512b0d 100644 --- a/test/test_detector.py +++ b/test/test_detector.py @@ -1,6 +1,7 @@ import os import unittest +import pytest import torch from pytorch_lightning.loggers import WandbLogger from torch import nn @@ -100,6 +101,10 @@ def test_model_init_heatmaps(self): self.assertTrue(torch.mean(heatmap).item() < 0.1) self.assertTrue(torch.var(heatmap).item() < 0.1) + # TODO: chcek if we can run it on gh actions as well. + IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true" + + @pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work in Github Actions atm") def test_checkpoint_loading(self): wandb_logger = WandbLogger(dir=get_wandb_log_dir_path(), mode="offline")