ModelCheckpoint does not save checkpoints

[Menion93]

This same code worked in the past version, but now it doesn't save the checkpoints anymore.

I tried with
MODEL_OUTPUT = 'example/hello'
MODEL_OUTPUT = 'example/hello/'
MODEL_OUTPUT = 'example/hello/weights'

with 'example/hello/' as an existing folder

checkpoint_callback = ModelCheckpoint(
        filepath='%s/' % MODEL_OUTPUT,
        save_best_only=True,
        verbose=True,
        monitor='val_loss',
        mode='min'
    )

    early_stop_callback = EarlyStopping(monitor='val_loss', patience=5)

    tt_logger = CMLLogger(LOG_DIR)

    trainer = Trainer(logger=tt_logger,
                      row_log_interval=2,
                      checkpoint_callback=checkpoint_callback,
                      early_stop_callback=early_stop_callback,
                      gradient_clip_val=0.5,
                      gpus=gpus,
                      check_val_every_n_epoch=1,
                      max_nb_epochs=99999,
                      train_percent_check=train_frac,
                      log_save_interval=5,
                     )

Desktop (please complete the following information):

• OS:linux/mac

[stas6626]

met this problem:) fixed by set train_percent_check=1

[Menion93]

Unfortunately I cannot set train_percent_check to 1 :(

[williamFalcon]

what train_frac are you using? are you using the latest version on master?

[Menion93]

The latest published release. I noticed now that train_frac is set to 1 in all of the examples above.

I forgot the mention that I'm subclassing a class that extends LightningModule as my model, in previous version this works fine.

[williamFalcon]

could you post full code or something? or code to reproduce?

[Menion93]

Sure, here are the train method and how I structured the class layout, I cannot post the network architecture but I don't think it is relevant at all

def fit_model(model, train_size, gpus=False):

    remove_folder_rec(LOG_DIR)
    remove_folder_rec(MODEL_OUTPUT)

    if gpus:
        gpus = str(1)
    else:
        gpus = None

    train_frac = min(1, 1 / (train_size / MAX_EXAMPLES_PER_EPOCHS))

    checkpoint_callback = ModelCheckpoint(
        filepath='%s/weights' % MODEL_OUTPUT,
        save_best_only=True,
        verbose=True,
        monitor='val_loss',
        mode='min'
    )

    early_stop_callback = EarlyStopping(monitor='val_loss', patience=5)

    tt_logger = CustomLogger(LOG_DIR)

    trainer = Trainer(logger=tt_logger,
                      row_log_interval=2,
                      checkpoint_callback=checkpoint_callback,
                      early_stop_callback=early_stop_callback,
                      gradient_clip_val=0.5,
                      gpus=gpus,
                      check_val_every_n_epoch=1,
                      max_nb_epochs=99999,
                      train_percent_check=train_frac,
                      log_save_interval=5,
                     )

    # Add error handling here
    # try:
    trainer.fit(model)
    # except Exception as e:
    #     print(str(e))
    #     print('early stopped')
    return trainer

class BaseModel(pl.LightningModule):

    def __init__(self,
                 batch_size,
                 gpu,
                 train_dataset=None,
                 val_dataset=None,
                 test_dataset=None):
        super(BaseModel, self).__init__()
        self.train_dataset = train_dataset
        self.val_dataset = val_dataset
        self.test_dataset = test_dataset
        self.gpu = gpu
        self.batch_size = batch_size

    def forward(self, inp, training=False):
        return NotImplemented

    def training_step(self, batch, batch_nb):
        return NotImplemented

    def validation_step(self, batch, batch_nb):
        return NotImplemented

    def validation_end(self, outputs):
        return NotImplemented

    def test_step(self, batch, batch_nb):
        return NotImplemented

    def test_end(self, outputs):
        return NotImplemented

    def configure_optimizers(self):
        return NotImplemented

    @pl.data_loader
    def train_dataloader(self):
        return DataLoader(self.train_dataset, shuffle=True, pin_memory=self.gpu, batch_size=self.batch_size)

    @pl.data_loader
    def val_dataloader(self):
        return DataLoader(self.val_dataset, shuffle=True, pin_memory=self.gpu, batch_size=self.batch_size)

    @pl.data_loader
    def test_dataloader(self):
        return DataLoader(self.test_dataset, shuffle=True, pin_memory=self.gpu, batch_size=self.batch_size)

class TextCNN(BaseModel):

    def __init__(self,                 
                 gpu=False,
                 train_dataset=None,
                 val_dataset=None,
                 test_dataset=None):
        super(TextCNN, self).__init__(
            1, gpu, train_dataset, val_dataset, test_dataset)

        ## Layers initializations is defined here...

    def forward(self, inp, training=False):
       ...
       ...
        return output

    def training_step(self, batch, batch_nb):
        x = torch.squeeze(batch['x'], dim=0).float()
        y = torch.squeeze(batch['y'], dim=0).long()

        output = self.forward(x, training=True)
        loss = self.loss(output, y)
        acc = accuracy(output, y)

        return {
            'loss': loss,
            'progress_bar': {
                'train_loss': loss,
                'train_acc': acc
            },
            'log': {
                'train_loss': loss,
                'train_acc': acc
            }
        }

    def validation_step(self, batch, batch_nb):
        x = torch.squeeze(batch['x'], dim=0).float()
        y = torch.squeeze(batch['y'], dim=0).long()

        output = self.forward(x)
        return {'batch_val_loss': self.loss(output, y),
                'batch_val_acc': accuracy(output, y)}

    def validation_end(self, outputs):
        avg_loss = torch.stack([x['batch_val_loss'] for x in outputs]).mean()
        avg_acc = torch.stack([x['batch_val_acc'] for x in outputs]).mean()

        return {
            'log': {
                'val_loss': avg_loss,
                'val_acc': avg_acc,
            },
            'progress_bar': {
                'val_loss': avg_loss,
                'val_acc': avg_acc
            }
        }

    def test_step(self, batch, batch_nb):
        x = torch.squeeze(batch['x'], dim=0).float()
        y = torch.squeeze(batch['y'], dim=0).long()

        output = self.forward(x)
        return {'batch_test_loss': self.loss(output, y),
                'batch_test_acc': accuracy(output, y)}

    def test_end(self, outputs):
        avg_loss = torch.stack([x['batch_test_loss'] for x in outputs]).mean()
        avg_acc = torch.stack([x['batch_test_acc'] for x in outputs]).mean()

        return {
            'log': {
                'test_loss': avg_loss,
                'test_acc': avg_acc,
            },
            'progress_bar': {
                'test_loss': avg_loss,
                'test_acc': avg_acc
            }
        }

    def configure_optimizers(self):
        return [torch.optim.Adam(self.parameters(), lr=3e-04)]

[festeh]

Well, you seem to monitor val_loss in your callback, but you do not return it in the separate key in validation_end, if you're using latest version, this may be a problem, check sources https://github.com/williamFalcon/pytorch-lightning/blob/0fac2d64cf4c6c31d3b21605a272ace01d0c51a2/pytorch_lightning/trainer/trainer.py#L1304

[Menion93]

I see, I need to include the metrics in the parent node of the object, thank you

like this:

{
    'log': {
        'val_loss': avg_loss,
        'val_acc': avg_acc,
    },
   'progress_bar': {
        'val_loss': avg_loss,
        'val_acc': avg_acc
    },
    'val_loss': avg_loss,
    'val_acc': avg_acc
}

[williamFalcon]

hadn't realized that wasn't the default. i agree that makes more intuitive sense. fixed on master