From 6032f4d9ed10e622e126a5601495e8ada2688dcd Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sat, 22 Jan 2022 21:33:17 -0500 Subject: [PATCH 01/12] Support machines without CUDA --- ffcv/loader/epoch_iterator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ffcv/loader/epoch_iterator.py b/ffcv/loader/epoch_iterator.py index 4bc1b9bd..7511de2e 100644 --- a/ffcv/loader/epoch_iterator.py +++ b/ffcv/loader/epoch_iterator.py @@ -46,9 +46,8 @@ def __init__(self, loader: 'Loader', order: Sequence[int]): self.memory_bank_per_stage = defaultdict(list) - if IS_CUDA: - self.cuda_streams = [ch.cuda.Stream() - for _ in range(self.loader.batches_ahead + 2)] + self.cuda_streams = [(ch.cuda.Stream() if IS_CUDA else None) + for _ in range(self.loader.batches_ahead + 2)] # Allocate all the memory memory_allocations = {} From 1d66ed3d18dad09696857578ad14dab1cb7d7cc6 Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sat, 22 Jan 2022 21:43:07 -0500 Subject: [PATCH 02/12] Fix setup.py - Resolves #88 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 075765d5..7d94fff7 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,8 @@ def pkgconfig(package, kw): description=' FFCV: Fast Forward Computer Vision ', author='MadryLab', author_email='leclerc@mit.edu', - url='https://github.com/MadryLab/fastercv', + url='https://github.com/libffcv/ffcv', + license_files = ('LICENSE.txt',), packages=find_packages(), long_description=long_description, long_description_content_type='text/markdown', From 6de7c173fe47c71f00ab46c62d7df2167760cac3 Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sat, 22 Jan 2022 23:32:17 -0500 Subject: [PATCH 03/12] Fix Distributed + custom indices - Resolves #90 --- ffcv/traversal_order/random.py | 2 +- ffcv/traversal_order/sequential.py | 2 +- tests/test_traversal_orders.py | 54 ++++++++++++++++++++---------- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/ffcv/traversal_order/random.py b/ffcv/traversal_order/random.py index cef4ac24..25bc1708 100644 --- a/ffcv/traversal_order/random.py +++ b/ffcv/traversal_order/random.py @@ -24,4 +24,4 @@ def sample_order(self, epoch: int) -> Sequence[int]: self.sampler.set_epoch(epoch) - return np.array(list(self.sampler)) + return self.indices[np.array(list(self.sampler))] diff --git a/ffcv/traversal_order/sequential.py b/ffcv/traversal_order/sequential.py index 632a0059..c5bba224 100644 --- a/ffcv/traversal_order/sequential.py +++ b/ffcv/traversal_order/sequential.py @@ -27,4 +27,4 @@ def sample_order(self, epoch: int) -> Sequence[int]: self.sampler.set_epoch(epoch) - return np.array(list(self.sampler)) + return self.indices[np.array(list(self.sampler))] diff --git a/tests/test_traversal_orders.py b/tests/test_traversal_orders.py index 9b771a95..4a632bbd 100644 --- a/tests/test_traversal_orders.py +++ b/tests/test_traversal_orders.py @@ -29,24 +29,29 @@ def __getitem__(self, index): return (index, np.sin(np.array([index])).view(' 1: - init_process_group('nccl', sync_url, rank=rank, world_size=world_size) + init_process_group('gloo', sync_url, rank=rank, world_size=world_size) loader = Loader(fname, 8, num_workers=2, order=order, drop_last=False, - distributed=world_size > 1) + distributed=world_size > 1, indices=indices) result = [] for _ in range(3): content = np.concatenate([x[0].numpy().reshape(-1).copy() for x in loader]) result.append(content) result = np.stack(result) + np.save(path.join(out_folder, f"result-{rank}.npy"), result) -def prep_and_run_test(num_workers, order): +def prep_and_run_test(num_workers, order, with_indices=False): length = 600 + indices = None + if with_indices: + indices = np.random.choice(length, length//2, replace=False) + with TemporaryDirectory() as folder: name = path.join(folder, 'dataset.beton') sync_file = path.join(folder, 'share') @@ -58,7 +63,7 @@ def prep_and_run_test(num_workers, order): writer.from_indexed_dataset(dataset) - args = (num_workers, name, order, sync_file, folder) + args = (num_workers, name, order, sync_file, folder, indices) if num_workers > 1: spawn(process_work, nprocs=num_workers, args=args) else: @@ -71,19 +76,22 @@ def prep_and_run_test(num_workers, order): results = np.concatenate(results, 1) + # For each epoch for i in range(results.shape[0]): - if order == OrderOption.SEQUENTIAL and i < results.shape[0] - 1: - assert_that((results[i] == results[i + 1]).all()).is_true() - if order != OrderOption.SEQUENTIAL and i < results.shape[0] - 1: - assert_that((results[i] == results[i + 1]).all()).is_false() - - epoch_content = Counter(results[i]) - indices_gotten = np.array(sorted(list(epoch_content.keys()))) - assert_that(np.all(np.arange(length) == indices_gotten)).is_true() - assert_that(min(epoch_content.values())).is_equal_to(1) - assert_that(max(epoch_content.values())).is_less_than_or_equal_to(2) - - + if not with_indices: + if order == OrderOption.SEQUENTIAL and i < results.shape[0] - 1: + assert_that((results[i] == results[i + 1]).all()).is_true() + if order != OrderOption.SEQUENTIAL and i < results.shape[0] - 1: + assert_that((results[i] == results[i + 1]).all()).is_false() + + epoch_content = Counter(results[i]) + indices_gotten = np.array(sorted(list(epoch_content.keys()))) + assert_that(np.all(np.arange(length) == indices_gotten)).is_true() + assert_that(min(epoch_content.values())).is_equal_to(1) + assert_that(max(epoch_content.values())).is_less_than_or_equal_to(2) + else: + assert_that(set(results[i])).is_equal_to(set(indices)) + def test_traversal_sequential_1(): prep_and_run_test(1, OrderOption.SEQUENTIAL) @@ -123,3 +131,15 @@ def test_traversal_quasirandom_3(): @pytest.mark.skip() def test_traversal_quasirandom_4(): prep_and_run_test(4, OrderOption.QUASI_RANDOM) + +def test_traversal_sequential_distributed_with_indices(): + prep_and_run_test(2, OrderOption.SEQUENTIAL, True) + +def test_traversal_random_distributed_with_indices(): + prep_and_run_test(2, OrderOption.RANDOM, True) + +@pytest.mark.skip() +def test_traversal_quasi_random_distributed_with_indices(): + prep_and_run_test(2, OrderOption.QUASI_RANDOM, True) + +if __name__ == '__main__': \ No newline at end of file From 0485ee44535d48f89c3a030956b54dcf10936c01 Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sun, 23 Jan 2022 00:08:05 -0500 Subject: [PATCH 04/12] Make default seed random - We think that it makes more sense to have random runs by default --- ffcv/loader/loader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ffcv/loader/loader.py b/ffcv/loader/loader.py index 4fcc8678..b41192e8 100644 --- a/ffcv/loader/loader.py +++ b/ffcv/loader/loader.py @@ -93,7 +93,7 @@ def __init__(self, os_cache: bool = DEFAULT_OS_CACHE, order: ORDER_TYPE = OrderOption.SEQUENTIAL, distributed: bool = False, - seed: int = 0, # For ordering of samples + seed: int = None, # For ordering of samples indices: Sequence[int] = None, # For subset selection pipelines: Mapping[str, Sequence[Union[Operation, ch.nn.Module]]] = {}, @@ -103,6 +103,10 @@ def __init__(self, recompile: bool = False, # Recompile at every epoch ): + if seed is None: + tinfo = np.iinfo(np.int) + seed = np.random.randint(tinfo.min, tinfo.max) + # We store the original user arguments to be able to pass it to the # filtered version of the datasets self._args = { From 94ee0e21e7321931bf4a46db2902e6d8773c6ada Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sun, 23 Jan 2022 00:16:21 -0500 Subject: [PATCH 05/12] Put NCCL back in the traversal order test --- tests/test_traversal_orders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_traversal_orders.py b/tests/test_traversal_orders.py index 4a632bbd..05955478 100644 --- a/tests/test_traversal_orders.py +++ b/tests/test_traversal_orders.py @@ -32,7 +32,7 @@ def __getitem__(self, index): def process_work(rank, world_size, fname, order, sync_fname, out_folder, indices): sync_url = f'file://{sync_fname}' if world_size > 1: - init_process_group('gloo', sync_url, rank=rank, world_size=world_size) + init_process_group('nccl', sync_url, rank=rank, world_size=world_size) loader = Loader(fname, 8, num_workers=2, order=order, drop_last=False, distributed=world_size > 1, indices=indices) From bcd439c8cd1e99c8dbd2811f0f8aa2b26aa8b657 Mon Sep 17 00:00:00 2001 From: Andrew Ilyas Date: Sun, 23 Jan 2022 00:22:22 -0500 Subject: [PATCH 06/12] failing test --- tests/test_augmentations.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_augmentations.py b/tests/test_augmentations.py index 0d8776b3..b093f208 100644 --- a/tests/test_augmentations.py +++ b/tests/test_augmentations.py @@ -13,7 +13,7 @@ from ffcv.fields import IntField, RGBImageField from ffcv.loader import Loader from ffcv.pipeline.compiler import Compiler -from ffcv.transforms import Squeeze, Cutout, ToTensor, ToDevice, Poison +from ffcv.transforms import Squeeze, Cutout, ToTensor, Poison, RandomHorizontalFlip def run_test(length, pipeline, compile): my_dataset = Subset(CIFAR10(root='/tmp', train=True, download=True), range(length)) @@ -43,6 +43,13 @@ def run_test(length, pipeline, compile): assert_that(tot_indices).is_equal_to(len(my_dataset)) assert_that(tot_images).is_equal_to(len(my_dataset)) +def test_flip(): + run_test(100, [ + SimpleRGBImageDecoder(), + RandomHorizontalFlip(1.0), + ToTensor() + ], True) + def test_cutout(): run_test(100, [ SimpleRGBImageDecoder(), From a80723ec4aa3a9e5e1b159c1284001eb2191f4c8 Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sun, 23 Jan 2022 00:24:27 -0500 Subject: [PATCH 07/12] Fix Depreciation warning --- ffcv/loader/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffcv/loader/loader.py b/ffcv/loader/loader.py index b41192e8..ea32ba62 100644 --- a/ffcv/loader/loader.py +++ b/ffcv/loader/loader.py @@ -104,7 +104,7 @@ def __init__(self, ): if seed is None: - tinfo = np.iinfo(np.int) + tinfo = np.iinfo(int) seed = np.random.randint(tinfo.min, tinfo.max) # We store the original user arguments to be able to pass it to the From b23a8431222c4fecd0ed86e029ea74590e725169 Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sun, 23 Jan 2022 00:24:51 -0500 Subject: [PATCH 08/12] Fix syntax error in test_traversal_orders --- tests/test_traversal_orders.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_traversal_orders.py b/tests/test_traversal_orders.py index 05955478..21c00b6e 100644 --- a/tests/test_traversal_orders.py +++ b/tests/test_traversal_orders.py @@ -140,6 +140,4 @@ def test_traversal_random_distributed_with_indices(): @pytest.mark.skip() def test_traversal_quasi_random_distributed_with_indices(): - prep_and_run_test(2, OrderOption.QUASI_RANDOM, True) - -if __name__ == '__main__': \ No newline at end of file + prep_and_run_test(2, OrderOption.QUASI_RANDOM, True) \ No newline at end of file From cf3431be5d009ecb53038c4736ef3c3d61e95c3a Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sun, 23 Jan 2022 00:40:33 -0500 Subject: [PATCH 09/12] Fix last batch when drop_last = False --- ffcv/loader/epoch_iterator.py | 2 +- tests/test_augmentations.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ffcv/loader/epoch_iterator.py b/ffcv/loader/epoch_iterator.py index 7511de2e..b54fa96b 100644 --- a/ffcv/loader/epoch_iterator.py +++ b/ffcv/loader/epoch_iterator.py @@ -135,7 +135,7 @@ def run_pipeline(self, b_ix, batch_indices, batch_slot, cuda_event): if first_stage: first_stage = False self.memory_context.end_batch(b_ix) - return tuple(args) + return tuple(x[:len(batch_indices)] for x in args) def __next__(self): result = self.output_queue.get() diff --git a/tests/test_augmentations.py b/tests/test_augmentations.py index b093f208..d0cd5140 100644 --- a/tests/test_augmentations.py +++ b/tests/test_augmentations.py @@ -76,4 +76,4 @@ def test_poison(): ], False) if __name__ == '__main__': - test_poison() + test_flip() From 4fb7525c536c5a7cb30ff3f19ae2c13970c9cab5 Mon Sep 17 00:00:00 2001 From: Guillaume Leclerc Date: Sun, 23 Jan 2022 00:47:04 -0500 Subject: [PATCH 10/12] Bump version --- ffcv/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ffcv/__init__.py b/ffcv/__init__.py index cbbcbd22..7fdfb1d6 100644 --- a/ffcv/__init__.py +++ b/ffcv/__init__.py @@ -1,5 +1,5 @@ from .loader import Loader from .writer import DatasetWriter -__version__ = '0.0.2' +__version__ = '0.0.3rc1' __all__ = ['Loader'] diff --git a/setup.py b/setup.py index 7d94fff7..978a6df9 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def pkgconfig(package, kw): **extension_kwargs) setup(name='ffcv', - version='0.0.2', + version='0.0.3rc1', description=' FFCV: Fast Forward Computer Vision ', author='MadryLab', author_email='leclerc@mit.edu', From f7e2e472a19dbf703c19dfc1d27d1763b53d4b97 Mon Sep 17 00:00:00 2001 From: Andrew Ilyas Date: Sun, 23 Jan 2022 11:43:27 -0500 Subject: [PATCH 11/12] fixing tests --- examples/imagenet-example | 2 +- ffcv/loader/loader.py | 10 +++++++--- tests/test_image_pipeline.py | 15 +++++++++------ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/examples/imagenet-example b/examples/imagenet-example index d394723e..f134cbff 160000 --- a/examples/imagenet-example +++ b/examples/imagenet-example @@ -1 +1 @@ -Subproject commit d394723e41e023017562df86a0e95c04be5cd119 +Subproject commit f134cbfff7f590954edc5c24275444b7dd2f57f6 diff --git a/ffcv/loader/loader.py b/ffcv/loader/loader.py index ea32ba62..21cdd104 100644 --- a/ffcv/loader/loader.py +++ b/ffcv/loader/loader.py @@ -103,9 +103,13 @@ def __init__(self, recompile: bool = False, # Recompile at every epoch ): - if seed is None: - tinfo = np.iinfo(int) - seed = np.random.randint(tinfo.min, tinfo.max) + if distributed and order == OrderOption.RANDOM and (seed is None): + print('Warning: no ordering seed was specified with distributed=True. ' + 'Setting seed to 0 to match PyTorch distributed sampler.') + seed = 0 + elif seed is None: + tinfo = np.iinfo('int32') + seed = np.random.randint(0, tinfo.max) # We store the original user arguments to be able to pass it to the # filtered version of the datasets diff --git a/tests/test_image_pipeline.py b/tests/test_image_pipeline.py index 289ba638..7338b772 100644 --- a/tests/test_image_pipeline.py +++ b/tests/test_image_pipeline.py @@ -38,6 +38,7 @@ def create_and_validate(length, mode='raw', reversed=False): with NamedTemporaryFile() as handle: name = handle.name + print(name) fields = { 'index': IntField(), @@ -68,7 +69,9 @@ def create_and_validate(length, mode='raw', reversed=False): if mode == 'raw': assert_that(ch.all((image == (i % 255)).reshape(-1))).is_true() else: + print('Here') assert_that(ch.all((image == (i % 255)).reshape(-1))).is_true() + print('Here 2', ch.all((image == (i % 255)).reshape(-1))) def make_and_read_cifar_subset(length): my_dataset = Subset(CIFAR10(root='/tmp', train=True, download=True), range(length)) @@ -90,14 +93,14 @@ def make_and_read_cifar_subset(length): for index, images in loader: pass -def test_cifar_subset(): - make_and_read_cifar_subset(200) +# def test_cifar_subset(): + # make_and_read_cifar_subset(200) -def test_simple_raw_image_pipeline(): - create_and_validate(500, 'raw', False) +# def test_simple_raw_image_pipeline(): +# create_and_validate(500, 'raw', False) -def test_simple_raw_image_pipeline_rev(): - create_and_validate(500, 'raw', True) +# def test_simple_raw_image_pipeline_rev(): + # create_and_validate(500, 'raw', True) def test_simple_jpg_image_pipeline(): create_and_validate(500, 'jpg', False) From 405c096002527039146ed34da0675874ede9b2c1 Mon Sep 17 00:00:00 2001 From: Andrew Ilyas Date: Sun, 23 Jan 2022 17:23:03 -0500 Subject: [PATCH 12/12] bring back tests --- tests/test_image_pipeline.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_image_pipeline.py b/tests/test_image_pipeline.py index 7338b772..289ba638 100644 --- a/tests/test_image_pipeline.py +++ b/tests/test_image_pipeline.py @@ -38,7 +38,6 @@ def create_and_validate(length, mode='raw', reversed=False): with NamedTemporaryFile() as handle: name = handle.name - print(name) fields = { 'index': IntField(), @@ -69,9 +68,7 @@ def create_and_validate(length, mode='raw', reversed=False): if mode == 'raw': assert_that(ch.all((image == (i % 255)).reshape(-1))).is_true() else: - print('Here') assert_that(ch.all((image == (i % 255)).reshape(-1))).is_true() - print('Here 2', ch.all((image == (i % 255)).reshape(-1))) def make_and_read_cifar_subset(length): my_dataset = Subset(CIFAR10(root='/tmp', train=True, download=True), range(length)) @@ -93,14 +90,14 @@ def make_and_read_cifar_subset(length): for index, images in loader: pass -# def test_cifar_subset(): - # make_and_read_cifar_subset(200) +def test_cifar_subset(): + make_and_read_cifar_subset(200) -# def test_simple_raw_image_pipeline(): -# create_and_validate(500, 'raw', False) +def test_simple_raw_image_pipeline(): + create_and_validate(500, 'raw', False) -# def test_simple_raw_image_pipeline_rev(): - # create_and_validate(500, 'raw', True) +def test_simple_raw_image_pipeline_rev(): + create_and_validate(500, 'raw', True) def test_simple_jpg_image_pipeline(): create_and_validate(500, 'jpg', False)