From 841a238310ce63834a4276633368aef321840342 Mon Sep 17 00:00:00 2001 From: NicEggert Date: Sat, 10 Aug 2019 13:21:54 -0500 Subject: [PATCH] When running DDP without DistributedSampler, throw warning instead of exception --- pytorch_lightning/models/trainer.py | 9 ++++++--- tests/test_models.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index 1c646a1221c9a..1955dd3e99bbb 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -444,8 +444,9 @@ def get_dataloaders(self, model): if self.use_ddp and not isinstance(self.tng_dataloader.sampler, DistributedSampler): msg = """ -when using multiple gpus and multiple nodes you must pass - a DistributedSampler to DataLoader(sampler). +You're using multiple gpus and multiple nodes without using a DistributedSampler +to assign a subset of your data to each process. To silence this warning, pass a +DistributedSampler to your DataLoader. ie: this: dataset = myDataset() @@ -455,8 +456,10 @@ def get_dataloaders(self, model): dataset = myDataset() dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) dataloader = Dataloader(dataset, sampler=dist_sampler) + +If you want each process to load the full dataset, ignore this warning. """ - raise MisconfigurationException(msg) + warnings.warn(msg) # ----------------------------- # MODEL TRAINING diff --git a/tests/test_models.py b/tests/test_models.py index cad55ff372775..896eb88490913 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -665,7 +665,7 @@ def test_ddp_sampler_error(): use_amp=True ) - with pytest.raises(MisconfigurationException): + with pytest.warns(UserWarning): trainer.get_dataloaders(model) clear_save_dir()