DDP Logdir Multiple Runs Bug

[lukasfolle]

🐛 Bug

Using accelerator="ddp" and running a pl Experiment mulitple times each time creating a new Trainer, the log version number is handled in a wrong way.
Instead of creating folder "version_2" after running "version_1" a folder with name "2" is created.
To fix this, after the training, os.environ.pop("PL_EXP_VERSION") has to be executed.

Please reproduce using the BoringModel and post here

https://colab.research.google.com/drive/1GFQ6doCWl_aTdBaBVOgTf1dTs_qZV0sg?usp=sharing

Expected behavior

Increase "version_1" to "version_2".

[github-actions]

Hi! thanks for your contribution!, great first issue!

[lukasfolle]

@Borda Any updates on this?

[awaelchli]

@lukasfolle is this still a problem? You provided a colab example but ddp is not supported in notebooks.
Note that the behavior is different when you run your script multiple times vs. when you call Trainer.fit multiple times in the same script.

[lukasfolle]

Please have a look at the code in the Colab. The issues arises when looping over trainer.fit().

[awaelchli]

Looking into fixing this now.
Repro:

import os
import torch
from torch.utils.data import Dataset
from pytorch_lightning import LightningModule, Trainer


class RandomDataset(Dataset):

    def __init__(self, size, length):
        self.len = length
        self.data = torch.randn(length, size)

    def __getitem__(self, index):
        return self.data[index]

    def __len__(self):
        return self.len


class BoringModel(LightningModule):

    def __init__(self):
        super().__init__()
        self.layer = torch.nn.Linear(32, 2)

    def forward(self, x):
        return self.layer(x)

    def loss(self, batch, prediction):
        # An arbitrary loss to have a loss that updates the model weights during `Trainer.fit` calls
        return torch.nn.functional.mse_loss(prediction, torch.ones_like(prediction))

    def training_step(self, batch, batch_idx):
        output = self.layer(batch)
        loss = self.loss(batch, output)
        return {"loss": loss}

    def training_step_end(self, training_step_outputs):
        return training_step_outputs

    def training_epoch_end(self, outputs) -> None:
        torch.stack([x["loss"] for x in outputs]).mean()

    def validation_step(self, batch, batch_idx):
        output = self.layer(batch)
        loss = self.loss(batch, output)
        return {"x": loss}

    def validation_epoch_end(self, outputs) -> None:
        torch.stack([x['x'] for x in outputs]).mean()

    def test_step(self, batch, batch_idx):
        output = self.layer(batch)
        loss = self.loss(batch, output)
        self.log('fake_test_acc', loss)
        return {"y": loss}

    def test_epoch_end(self, outputs) -> None:
        torch.stack([x["y"] for x in outputs]).mean()

    def configure_optimizers(self):
        optimizer = torch.optim.SGD(self.layer.parameters(), lr=0.1)
        lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1)
        return [optimizer], [lr_scheduler]


def run():
    train_data = torch.utils.data.DataLoader(RandomDataset(32, 64), batch_size=2, num_workers=0)
    val_data = torch.utils.data.DataLoader(RandomDataset(32, 64), batch_size=2, num_workers=0)
    test_data = torch.utils.data.DataLoader(RandomDataset(32, 64), batch_size=2, num_workers=0)

    for _ in range(2):
        model = BoringModel()
        trainer = Trainer(
            max_epochs=1,
            progress_bar_refresh_rate=20,
            accelerator="ddp",
            gpus=2
        )
        trainer.fit(model, train_data, val_data)
        trainer.test(test_dataloaders=test_data)
        print(os.environ.get("PL_EXP_VERSION"))
        del trainer, model


if __name__ == '__main__':
    run()