From 2d84076adbc760073fc6bfc04c767d6315dce627 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 20 Dec 2021 11:16:52 +0100 Subject: [PATCH 01/10] add prototype dataset for oxford-iiit-pet --- .../prototype/datasets/_builtin/__init__.py | 1 + .../_builtin/oxford-iiit-pet.categories | 37 +++++ .../datasets/_builtin/oxford_iiit_pet.py | 150 ++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories create mode 100644 torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py diff --git a/torchvision/prototype/datasets/_builtin/__init__.py b/torchvision/prototype/datasets/_builtin/__init__.py index 62abc3119f6..9f67e0483de 100644 --- a/torchvision/prototype/datasets/_builtin/__init__.py +++ b/torchvision/prototype/datasets/_builtin/__init__.py @@ -4,6 +4,7 @@ from .coco import Coco from .imagenet import ImageNet from .mnist import MNIST, FashionMNIST, KMNIST, EMNIST, QMNIST +from .oxford_iiit_pet import OxfordIITPet from .sbd import SBD from .semeion import SEMEION from .voc import VOC diff --git a/torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories b/torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories new file mode 100644 index 00000000000..f97035695fe --- /dev/null +++ b/torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories @@ -0,0 +1,37 @@ +Abyssinian +British Shorthair +Chihuahua +Egyptian Mau +English Cocker Spaniel +English Setter +German Shorthaired +Great Pyrenees +Havanese +Japanese Chin +Keeshond +American Bulldog +Leonberger +Maine Coon +Miniature Pinscher +Newfoundland +Persian +Pomeranian +Pug +Ragdoll +Russian Blue +Saint Bernard +American Pit Bull Terrier +Samoyed +Scottish Terrier +Shiba Inu +Siamese +Sphynx +Staffordshire Bull Terrier +Wheaten Terrier +Yorkshire Terrier +Basset Hound +Beagle +Bengal +Birman +Bombay +Boxer diff --git a/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py b/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py new file mode 100644 index 00000000000..c0c4745a3f0 --- /dev/null +++ b/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py @@ -0,0 +1,150 @@ +import functools +import io +import pathlib +from typing import Any, Callable, Dict, List, Optional, Tuple + +import torch +from torchdata.datapipes.iter import IterDataPipe, Mapper, Filter, IterKeyZipper, Demultiplexer, CSVDictParser +from torchvision.prototype.datasets.utils import ( + Dataset, + DatasetConfig, + DatasetInfo, + HttpResource, + OnlineResource, + DatasetType, +) +from torchvision.prototype.datasets.utils._internal import ( + INFINITE_BUFFER_SIZE, + hint_sharding, + hint_shuffling, + getitem, + path_accessor, + path_comparator, +) +from torchvision.prototype.features import Label + + +class OxfordIITPet(Dataset): + def _make_info(self) -> DatasetInfo: + return DatasetInfo( + "oxford-iiit-pet", + type=DatasetType.IMAGE, + homepage="https://www.robots.ox.ac.uk/~vgg/data/pets/", + valid_options=dict( + # FIXME + split=("trainval", "test", "train"), + ), + ) + + def resources(self, config: DatasetConfig) -> List[OnlineResource]: + images = HttpResource( + "https://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz", + sha256="67195c5e1c01f1ab5f9b6a5d22b8c27a580d896ece458917e61d459337fa318d", + decompress=True, + ) + anns = HttpResource( + "https://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz", + sha256="52425fb6de5c424942b7626b428656fcbd798db970a937df61750c0f1d358e91", + decompress=True, + ) + return [images, anns] + + def _classify_anns(self, data: Tuple[str, Any]) -> Optional[int]: + return { + "annotations": 0, + "trimaps": 1, + }.get(pathlib.Path(data[0]).parent.name) + + def _filter_images(self, data: Tuple[str, Any]) -> bool: + return pathlib.Path(data[0]).suffix == ".jpg" + + def _filter_segmentations(self, data: Tuple[str, Any]) -> bool: + return not pathlib.Path(data[0]).name.startswith(".") + + def _decode_classification_data(self, data: Dict[str, str]) -> Dict[str, Any]: + label_idx = int(data["label"]) - 1 + return dict( + label=Label(label_idx, category=self.info.categories[label_idx]), + species="cat" if data["species"] == "1" else "dog", + ) + + def _collate_and_decode_sample( + self, + data: Tuple[Tuple[Dict[str, str], Tuple[str, io.IOBase]], Tuple[str, io.IOBase]], + *, + decoder: Optional[Callable[[io.IOBase], torch.Tensor]], + ) -> Dict[str, Any]: + ann_data, image_data = data + classification_data, segmentation_data = ann_data + segmentation_path, segmentation_buffer = segmentation_data + image_path, image_buffer = image_data + + return dict( + self._decode_classification_data(classification_data), + segmentation_path=segmentation_path, + segmentation=decoder(segmentation_buffer) if decoder else segmentation_buffer, + image_path=image_path, + image=decoder(image_buffer) if decoder else image_buffer, + ) + + def _make_datapipe( + self, + resource_dps: List[IterDataPipe], + *, + config: DatasetConfig, + decoder: Optional[Callable[[io.IOBase], torch.Tensor]], + ) -> IterDataPipe[Dict[str, Any]]: + images_dp, anns_dp = resource_dps + + images_dp = Filter(images_dp, self._filter_images) + + split_and_classification_dp, segmentations_dp = Demultiplexer( + anns_dp, + 2, + self._classify_anns, + drop_none=True, + buffer_size=INFINITE_BUFFER_SIZE, + ) + + split_and_classification_dp = Filter( + split_and_classification_dp, path_comparator("name", f"{config.split}.txt") + ) + split_and_classification_dp = CSVDictParser( + split_and_classification_dp, fieldnames=("image_id", "label", "species"), delimiter=" " + ) + split_and_classification_dp = hint_sharding(split_and_classification_dp) + split_and_classification_dp = hint_shuffling(split_and_classification_dp) + + segmentations_dp = Filter(segmentations_dp, self._filter_segmentations) + + anns_dp = IterKeyZipper( + split_and_classification_dp, + segmentations_dp, + key_fn=getitem("image_id"), + ref_key_fn=path_accessor("stem"), + buffer_size=INFINITE_BUFFER_SIZE, + ) + + dp = IterKeyZipper( + anns_dp, + images_dp, + key_fn=getitem(0, "image_id"), + ref_key_fn=path_accessor("stem"), + buffer_size=INFINITE_BUFFER_SIZE, + ) + return Mapper(dp, functools.partial(self._collate_and_decode_sample, decoder=decoder)) + + def _filter_split_and_classification_anns(self, data: Tuple[str, Any]): + return self._classify_anns(data) == 0 + + def _generate_categories(self, root: pathlib.Path) -> List[str]: + config = self.default_config + dp = self.resources(config)[1].load(pathlib.Path(root) / self.name) + dp = Filter(dp, self._filter_split_and_classification_anns) + dp = Filter(dp, path_comparator("name", f"{config.split}.txt")) + dp = CSVDictParser(dp, fieldnames=("image_id", "label"), delimiter=" ") + raw_categories_and_labels = {(data["image_id"].rsplit("_", 1)[0], data["label"]) for data in dp} + raw_categories, _ = zip( + *sorted(raw_categories_and_labels, key=lambda raw_category_and_label: raw_category_and_label[1]) + ) + return [" ".join(part.title() for part in raw_category.split("_")) for raw_category in raw_categories] From 597d62c8ff11306ff3feb0229dd43d2f5553d33f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 20 Dec 2021 14:17:41 +0100 Subject: [PATCH 02/10] add old-style dataset --- docs/source/datasets.rst | 1 + torchvision/datasets/__init__.py | 2 + torchvision/datasets/oxford_iiit_pet.py | 114 ++++++++++++++++++ .../_builtin/oxford-iiit-pet.categories | 16 +-- .../datasets/_builtin/oxford_iiit_pet.py | 2 +- 5 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 torchvision/datasets/oxford_iiit_pet.py diff --git a/docs/source/datasets.rst b/docs/source/datasets.rst index 7f09ff245ca..56fbffb7694 100644 --- a/docs/source/datasets.rst +++ b/docs/source/datasets.rst @@ -58,6 +58,7 @@ You can also create your own datasets using the provided :ref:`base classes int: + return len(self._images) + + def __getitem__(self, idx: int) -> Tuple[Any, Any]: + image = Image.open(self._images[idx]).convert("RGB") + + target = [] + for t in self._target_type: + if t == "category": + target.append(self._labels[idx]) + else: # t == "segmentation" + target.append(Image.open(self._segmentations[idx])) + + if not target: + target = None + elif len(target) == 1: + target = target[0] + else: + target = tuple(target) + + if self.transforms: + image, target = self.transforms(image, target) + + return image, target + + @property + def _base_folder(self): + return os.path.join(self.root, type(self).__name__.lower()) + + @property + def _images_folder(self) -> str: + return os.path.join(self._base_folder, "images") + + @property + def _anns_folder(self) -> str: + return os.path.join(self._base_folder, "annotations") + + @property + def _segmentations_folder(self) -> str: + return os.path.join(self._anns_folder, "trimaps") + + def _check_exists(self) -> bool: + for folder in (self._images_folder, self._anns_folder): + if not (os.path.exists(folder) and os.path.isdir(folder)): + return False + else: + return True + + def _download(self) -> None: + if self._check_exists(): + return + + for url, md5 in self._RESOURCES: + download_and_extract_archive(url, download_root=self._base_folder, md5=md5) diff --git a/torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories b/torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories index f97035695fe..36d29465b04 100644 --- a/torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories +++ b/torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories @@ -1,4 +1,12 @@ Abyssinian +American Bulldog +American Pit Bull Terrier +Basset Hound +Beagle +Bengal +Birman +Bombay +Boxer British Shorthair Chihuahua Egyptian Mau @@ -9,7 +17,6 @@ Great Pyrenees Havanese Japanese Chin Keeshond -American Bulldog Leonberger Maine Coon Miniature Pinscher @@ -20,7 +27,6 @@ Pug Ragdoll Russian Blue Saint Bernard -American Pit Bull Terrier Samoyed Scottish Terrier Shiba Inu @@ -29,9 +35,3 @@ Sphynx Staffordshire Bull Terrier Wheaten Terrier Yorkshire Terrier -Basset Hound -Beagle -Bengal -Birman -Bombay -Boxer diff --git a/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py b/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py index c0c4745a3f0..036d3b161ac 100644 --- a/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py +++ b/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py @@ -145,6 +145,6 @@ def _generate_categories(self, root: pathlib.Path) -> List[str]: dp = CSVDictParser(dp, fieldnames=("image_id", "label"), delimiter=" ") raw_categories_and_labels = {(data["image_id"].rsplit("_", 1)[0], data["label"]) for data in dp} raw_categories, _ = zip( - *sorted(raw_categories_and_labels, key=lambda raw_category_and_label: raw_category_and_label[1]) + *sorted(raw_categories_and_labels, key=lambda raw_category_and_label: int(raw_category_and_label[1])) ) return [" ".join(part.title() for part in raw_category.split("_")) for raw_category in raw_categories] From d1b828aa78e0de54f577bc2dfd16dfa6694fef7d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 20 Dec 2021 14:55:29 +0100 Subject: [PATCH 03/10] add tests --- test/test_datasets.py | 57 +++++++++++++++++++++++++ torchvision/datasets/oxford_iiit_pet.py | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/test/test_datasets.py b/test/test_datasets.py index 761f11d77dc..a8dc1e8838c 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -1,5 +1,6 @@ import bz2 import contextlib +import csv import io import itertools import json @@ -2168,5 +2169,61 @@ def inject_fake_data(self, tmpdir, config): return num_sequences * (num_examples_per_sequence - 1) +class LFWPeopleTestCase(datasets_utils.DatasetTestCase): + DATASET_CLASS = datasets.OxfordIIITPet + FEATURE_TYPES = (PIL.Image.Image, (int, PIL.Image.Image, tuple, type(None))) + ADDITIONAL_CONFIGS = datasets_utils.combinations_grid( + split=("trainval", "test"), + target_type=("category", "segmentation", ("category", "segmentation")), + ) + + def inject_fake_data(self, tmpdir, config): + base_folder = os.path.join(tmpdir, "oxford-iiit-pet") + + classification_anns_meta = ( + dict(cls="Abyssinian", label=0, species="cat"), + dict(cls="Keeshond", label=18, species="dog"), + dict(cls="Yorkshire Terrier", label=37, species="dog"), + ) + split_and_classification_anns = [ + self._meta_to_split_and_classification_ann(meta, idx) + for meta, idx in itertools.product(classification_anns_meta, (1, 2, 10)) + ] + image_ids, *_ = zip(*split_and_classification_anns) + + image_files = datasets_utils.create_image_folder( + base_folder, "images", file_name_fn=lambda idx: f"{image_ids[idx]}.jpg", num_examples=len(image_ids) + ) + + anns_folder = os.path.join(base_folder, "annotations") + os.makedirs(anns_folder) + split_and_classification_anns_in_split = random.choices(split_and_classification_anns, k=len(image_ids) // 2) + with open(os.path.join(anns_folder, f"{config['split']}.txt"), "w", newline="") as file: + writer = csv.writer(file, delimiter=" ") + for split_and_classification_ann in split_and_classification_anns_in_split: + writer.writerow(split_and_classification_ann) + + segmentation_files = datasets_utils.create_image_folder( + anns_folder, "trimaps", file_name_fn=lambda idx: f"{image_ids[idx]}.png", num_examples=len(image_ids) + ) + + # The dataset has some rouge files + for path in image_files[:2]: + path.with_suffix(".mat").touch() + for path in segmentation_files: + path.with_name(f".{path.name}").touch() + + return len(split_and_classification_anns_in_split) + + def _meta_to_split_and_classification_ann(self, meta, idx): + image_id = [ + "_".join((str.upper if meta["species"] == "cat" else str.lower)(part) for part in meta["cls"].split()) + ] + class_id = str(meta["label"] + 1) + species = "1" if meta["species"] == "cat" else "2" + breed_id = "-1" + return (image_id, class_id, species, breed_id) + + if __name__ == "__main__": unittest.main() diff --git a/torchvision/datasets/oxford_iiit_pet.py b/torchvision/datasets/oxford_iiit_pet.py index 19ab10db194..57ca53d425d 100644 --- a/torchvision/datasets/oxford_iiit_pet.py +++ b/torchvision/datasets/oxford_iiit_pet.py @@ -85,7 +85,7 @@ def __getitem__(self, idx: int) -> Tuple[Any, Any]: @property def _base_folder(self): - return os.path.join(self.root, type(self).__name__.lower()) + return os.path.join(self.root, "oxford-iiit-pet") @property def _images_folder(self) -> str: From 17ece538cdfc884bb89356b36a58b0281471aae4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 20 Dec 2021 14:57:12 +0100 Subject: [PATCH 04/10] fix mypy --- torchvision/datasets/oxford_iiit_pet.py | 1 + torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/torchvision/datasets/oxford_iiit_pet.py b/torchvision/datasets/oxford_iiit_pet.py index 57ca53d425d..04c0078e711 100644 --- a/torchvision/datasets/oxford_iiit_pet.py +++ b/torchvision/datasets/oxford_iiit_pet.py @@ -71,6 +71,7 @@ def __getitem__(self, idx: int) -> Tuple[Any, Any]: else: # t == "segmentation" target.append(Image.open(self._segmentations[idx])) + target: Any if not target: target = None elif len(target) == 1: diff --git a/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py b/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py index 036d3b161ac..4e43613715e 100644 --- a/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py +++ b/torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py @@ -134,7 +134,7 @@ def _make_datapipe( ) return Mapper(dp, functools.partial(self._collate_and_decode_sample, decoder=decoder)) - def _filter_split_and_classification_anns(self, data: Tuple[str, Any]): + def _filter_split_and_classification_anns(self, data: Tuple[str, Any]) -> bool: return self._classify_anns(data) == 0 def _generate_categories(self, root: pathlib.Path) -> List[str]: From 7086f33e15b84a36c69c87404396d411e49478ae Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 21 Dec 2021 17:53:24 +0100 Subject: [PATCH 05/10] fix test --- test/test_datasets.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/test_datasets.py b/test/test_datasets.py index a8dc1e8838c..5012438a3d1 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -2169,7 +2169,7 @@ def inject_fake_data(self, tmpdir, config): return num_sequences * (num_examples_per_sequence - 1) -class LFWPeopleTestCase(datasets_utils.DatasetTestCase): +class OxfordIIITPetTestCase(datasets_utils.DatasetTestCase): DATASET_CLASS = datasets.OxfordIIITPet FEATURE_TYPES = (PIL.Image.Image, (int, PIL.Image.Image, tuple, type(None))) ADDITIONAL_CONFIGS = datasets_utils.combinations_grid( @@ -2216,9 +2216,12 @@ def inject_fake_data(self, tmpdir, config): return len(split_and_classification_anns_in_split) def _meta_to_split_and_classification_ann(self, meta, idx): - image_id = [ - "_".join((str.upper if meta["species"] == "cat" else str.lower)(part) for part in meta["cls"].split()) - ] + image_id = "_".join( + [ + *[(str.title if meta["species"] == "cat" else str.lower)(part) for part in meta["cls"].split()], + str(idx), + ] + ) class_id = str(meta["label"] + 1) species = "1" if meta["species"] == "cat" else "2" breed_id = "-1" From b5c6539bef49b56055a35d2ad352e45d73b26c00 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 22 Dec 2021 09:32:54 +0100 Subject: [PATCH 06/10] remove properties and use pathlib --- torchvision/datasets/oxford_iiit_pet.py | 34 +++++++++---------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/torchvision/datasets/oxford_iiit_pet.py b/torchvision/datasets/oxford_iiit_pet.py index 04c0078e711..c0b1fc297f4 100644 --- a/torchvision/datasets/oxford_iiit_pet.py +++ b/torchvision/datasets/oxford_iiit_pet.py @@ -1,5 +1,6 @@ import os import os.path +import pathlib from typing import Any, Callable, Optional, Union, Tuple from typing import Sequence @@ -26,12 +27,17 @@ def __init__( target_transform: Optional[Callable] = None, download: bool = True, ): - super().__init__(root, transforms=transforms, transform=transform, target_transform=target_transform) self._split = verify_str_arg(split, "split", ("trainval", "test")) if isinstance(target_type, str): target_type = [target_type] self._target_type = [verify_str_arg(t, "target_type", self._TARGET_TYPES) for t in target_type] + super().__init__(root, transforms=transforms, transform=transform, target_transform=target_transform) + self._base_folder = pathlib.Path(self.root) / "oxford-iiit-pet" + self._images_folder = self._base_folder / "images" + self._anns_folder = self._base_folder / "annotations" + self._segs_folder = self._anns_folder / "trimaps" + if download: self._download() @@ -40,7 +46,7 @@ def __init__( image_ids = [] self._labels = [] - with open(os.path.join(self._anns_folder, f"{self._split}.txt")) as file: + with open(self._anns_folder / f"{self._split}.txt") as file: for line in file: image_id, label, *_ = line.strip().split() image_ids.append(image_id) @@ -55,8 +61,8 @@ def __init__( ] self.class_to_idx = dict(zip(self.classes, range(len(self.classes)))) - self._images = [os.path.join(self._images_folder, f"{image_id}.jpg") for image_id in image_ids] - self._segmentations = [os.path.join(self._segmentations_folder, f"{image_id}.png") for image_id in image_ids] + self._images = [self._images_folder / f"{image_id}.jpg" for image_id in image_ids] + self._segs = [self._segs_folder / f"{image_id}.png" for image_id in image_ids] def __len__(self) -> int: return len(self._images) @@ -69,7 +75,7 @@ def __getitem__(self, idx: int) -> Tuple[Any, Any]: if t == "category": target.append(self._labels[idx]) else: # t == "segmentation" - target.append(Image.open(self._segmentations[idx])) + target.append(Image.open(self._segs[idx])) target: Any if not target: @@ -84,22 +90,6 @@ def __getitem__(self, idx: int) -> Tuple[Any, Any]: return image, target - @property - def _base_folder(self): - return os.path.join(self.root, "oxford-iiit-pet") - - @property - def _images_folder(self) -> str: - return os.path.join(self._base_folder, "images") - - @property - def _anns_folder(self) -> str: - return os.path.join(self._base_folder, "annotations") - - @property - def _segmentations_folder(self) -> str: - return os.path.join(self._anns_folder, "trimaps") - def _check_exists(self) -> bool: for folder in (self._images_folder, self._anns_folder): if not (os.path.exists(folder) and os.path.isdir(folder)): @@ -112,4 +102,4 @@ def _download(self) -> None: return for url, md5 in self._RESOURCES: - download_and_extract_archive(url, download_root=self._base_folder, md5=md5) + download_and_extract_archive(url, download_root=str(self._base_folder), md5=md5) From d28945c92e04c7540c1138fe1af731798219302b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 5 Jan 2022 14:13:09 +0100 Subject: [PATCH 07/10] target_type to target_types --- torchvision/datasets/oxford_iiit_pet.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/torchvision/datasets/oxford_iiit_pet.py b/torchvision/datasets/oxford_iiit_pet.py index c0b1fc297f4..4192d2a16b7 100644 --- a/torchvision/datasets/oxford_iiit_pet.py +++ b/torchvision/datasets/oxford_iiit_pet.py @@ -15,22 +15,24 @@ class OxfordIIITPet(VisionDataset): ("https://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz", "5c4f3ee8e5d25df40f4fd59a7f44e54c"), ("https://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz", "95a8c909bbe2e81eed6a22bccdf3f68f"), ) - _TARGET_TYPES = ("category", "segmentation") + _VALID_TARGET_TYPES = ("category", "segmentation") def __init__( self, root: str, split: str = "trainval", - target_type: Union[Sequence[str], str] = "category", + target_types: Union[Sequence[str], str] = "category", transforms: Optional[Callable] = None, transform: Optional[Callable] = None, target_transform: Optional[Callable] = None, download: bool = True, ): self._split = verify_str_arg(split, "split", ("trainval", "test")) - if isinstance(target_type, str): - target_type = [target_type] - self._target_type = [verify_str_arg(t, "target_type", self._TARGET_TYPES) for t in target_type] + if isinstance(target_types, str): + target_types = [target_types] + self._target_type = [ + verify_str_arg(target_type, "target_types", self._VALID_TARGET_TYPES) for target_type in target_types + ] super().__init__(root, transforms=transforms, transform=transform, target_transform=target_transform) self._base_folder = pathlib.Path(self.root) / "oxford-iiit-pet" @@ -71,8 +73,8 @@ def __getitem__(self, idx: int) -> Tuple[Any, Any]: image = Image.open(self._images[idx]).convert("RGB") target = [] - for t in self._target_type: - if t == "category": + for target_type in self._target_type: + if target_type == "category": target.append(self._labels[idx]) else: # t == "segmentation" target.append(Image.open(self._segs[idx])) From 21a05edc78ecec663868a78fc71cebd4d23141e2 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 5 Jan 2022 14:14:55 +0100 Subject: [PATCH 08/10] move target annotation --- test/test_datasets.py | 2 +- torchvision/datasets/oxford_iiit_pet.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_datasets.py b/test/test_datasets.py index 5012438a3d1..7b8e0a88d53 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -2207,7 +2207,7 @@ def inject_fake_data(self, tmpdir, config): anns_folder, "trimaps", file_name_fn=lambda idx: f"{image_ids[idx]}.png", num_examples=len(image_ids) ) - # The dataset has some rouge files + # The dataset has some rogue files for path in image_files[:2]: path.with_suffix(".mat").touch() for path in segmentation_files: diff --git a/torchvision/datasets/oxford_iiit_pet.py b/torchvision/datasets/oxford_iiit_pet.py index 4192d2a16b7..46950a0fe15 100644 --- a/torchvision/datasets/oxford_iiit_pet.py +++ b/torchvision/datasets/oxford_iiit_pet.py @@ -72,14 +72,13 @@ def __len__(self) -> int: def __getitem__(self, idx: int) -> Tuple[Any, Any]: image = Image.open(self._images[idx]).convert("RGB") - target = [] + target: Any = [] for target_type in self._target_type: if target_type == "category": target.append(self._labels[idx]) else: # t == "segmentation" target.append(Image.open(self._segs[idx])) - target: Any if not target: target = None elif len(target) == 1: From 1a0f0d911ab2a6e02397ce1af3d15c842cba90e5 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 5 Jan 2022 14:22:09 +0100 Subject: [PATCH 09/10] add docstring --- torchvision/datasets/oxford_iiit_pet.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/torchvision/datasets/oxford_iiit_pet.py b/torchvision/datasets/oxford_iiit_pet.py index 46950a0fe15..65c01761ae5 100644 --- a/torchvision/datasets/oxford_iiit_pet.py +++ b/torchvision/datasets/oxford_iiit_pet.py @@ -11,6 +11,26 @@ class OxfordIIITPet(VisionDataset): + """`Oxford-IIIT Pet Dataset `_. + + Args: + root (string): Root directory of the dataset. + split (string, optional): The dataset split, supports ``"trainval"`` (default) or ``"test"``. + target_types (string, sequence of strings, optional): Types of target to use. Can be ``category`` (default) or + ``segmentation``. Can also be a list to output a tuple with all specified target types. The types represent: + + - ``category`` (int): Label for one of the 37 pet categories. + - ``segmentation`` (PIL image): Segmentation trimap of the image. + + If empty, ``None`` will be returned as target. + + transform (callable, optional): A function/transform that takes in a PIL image and returns a transformed + version. E.g, ``transforms.RandomCrop``. + target_transform (callable, optional): A function/transform that takes in the target and transforms it. + download (bool, optional): If True, downloads the dataset from the internet and puts it into ``root/dtd``. If + dataset is already downloaded, it is not downloaded again. + """ + _RESOURCES = ( ("https://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz", "5c4f3ee8e5d25df40f4fd59a7f44e54c"), ("https://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz", "95a8c909bbe2e81eed6a22bccdf3f68f"), From 323108a5d84d6fc1b539eb555f54aca4144592c5 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 6 Jan 2022 16:13:17 +0100 Subject: [PATCH 10/10] fix test --- test/test_datasets.py | 5 +++-- torchvision/datasets/oxford_iiit_pet.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/test_datasets.py b/test/test_datasets.py index 7b8e0a88d53..3f07c551cda 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -2169,12 +2169,13 @@ def inject_fake_data(self, tmpdir, config): return num_sequences * (num_examples_per_sequence - 1) -class OxfordIIITPetTestCase(datasets_utils.DatasetTestCase): +class OxfordIIITPetTestCase(datasets_utils.ImageDatasetTestCase): DATASET_CLASS = datasets.OxfordIIITPet FEATURE_TYPES = (PIL.Image.Image, (int, PIL.Image.Image, tuple, type(None))) + ADDITIONAL_CONFIGS = datasets_utils.combinations_grid( split=("trainval", "test"), - target_type=("category", "segmentation", ("category", "segmentation")), + target_types=("category", "segmentation", ["category", "segmentation"], []), ) def inject_fake_data(self, tmpdir, config): diff --git a/torchvision/datasets/oxford_iiit_pet.py b/torchvision/datasets/oxford_iiit_pet.py index 65c01761ae5..f386169411d 100644 --- a/torchvision/datasets/oxford_iiit_pet.py +++ b/torchvision/datasets/oxford_iiit_pet.py @@ -50,7 +50,7 @@ def __init__( self._split = verify_str_arg(split, "split", ("trainval", "test")) if isinstance(target_types, str): target_types = [target_types] - self._target_type = [ + self._target_types = [ verify_str_arg(target_type, "target_types", self._VALID_TARGET_TYPES) for target_type in target_types ] @@ -93,10 +93,10 @@ def __getitem__(self, idx: int) -> Tuple[Any, Any]: image = Image.open(self._images[idx]).convert("RGB") target: Any = [] - for target_type in self._target_type: + for target_type in self._target_types: if target_type == "category": target.append(self._labels[idx]) - else: # t == "segmentation" + else: # target_type == "segmentation" target.append(Image.open(self._segs[idx])) if not target: