3D Grad cam on multiple images

[roshnikotian15]

Hi,I am using trying to integrate monai gradcam in my 3d resnet model which is based on pytorch lightning module for multiclass problem but I am not successful. Can you tell me what needs to be done and where?I am trying to use val data loader to see the gradcam result on images. Here is my code:

import sys

import monai
from monai.visualize import plot_2d_or_3d_image
import json
import os

import matplotlib.pyplot as plt
from copy import deepcopy

import numpy as np
import pytorch_lightning as pl
import torch
import torch.nn.functional as F
import pandas as pd
from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score, \
    classification_report

from torch.utils.data import DataLoader
sys.path.append(os.path.dirname(os.path.dirname(__file__)))

from data_managment.DataLoader import ArthritisDataLoader
from data_managment.Dataset import get_persistent_dataset
from data_managment.Processing import build_processing_pipeline
from data_managment.mri.Database import Database
from deep_learning.architectures.Resnet3D.model import load_pretrained_model, generate_model
from arthritis_utils.TorchUtils import stack_tensors


class DataModule(pl.LightningDataModule):
    def __init__(self, hparams):
        super().__init__()
        self.batch_size = hparams["batch_size"]
        self.dataset = None
        self.split = [0.7, 0.2, 0.1]
        self.db_length = 0
        self.num_workers = 12
        self.hparams = hparams
        self.transformation_settings = {}
        self.transformations = self.get_transformations()
      
    def setup(self, stage=None):
        
        self.dataset.data = [sample for sample in self.dataset.data if sample["label"] in [0, 1, 2]]
        self.db_length = len(self.dataset)


    @staticmethod
    def get_transformations():
        [..........................................

        ]
        return pipeline

    def train_dataloader(self) -> DataLoader:
        train_dataset = deepcopy(self.dataset)
        train_dataset.data = train_dataset.data[0:int(self.db_length * self.split[0])]
        return ArthritisDataLoader(train_dataset, shuffle=True, batch_size=self.batch_size,
                                   num_workers=self.num_workers)

    def val_dataloader(self) -> DataLoader:
        val_dataset = deepcopy(self.dataset)
        val_dataset.data = val_dataset.data[int(self.db_length * self.split[0]):
                                            int(self.db_length * self.split[0] + self.db_length * self.split[1])]
        return ArthritisDataLoader(val_dataset, shuffle=False, batch_size=self.batch_size,
                                   num_workers=self.num_workers)

    def test_dataloader(self) -> DataLoader:
        test_dataset = deepcopy(self.dataset)
        test_dataset.data = test_dataset.data[-int(self.db_length * self.split[2]):]
        return ArthritisDataLoader(test_dataset, shuffle=False, batch_size=self.batch_size,
                                   num_workers=self.num_workers)



class Model(pl.LightningModule):
    def __init__(self, hparams):
        super().__init__()
        self.hparams = hparams

        self.model = pretrained resnet 3d used
        self.gradcam = monai.visualize.GradCAM(
            nn_module=self.model,
            target_layers="class_layers.relu"
        )

    def forward(self, volume):
        return self.model(volume)


    def configure_optimizers(self):
        return torch.optim.Adam(filter(lambda p: p.requires_grad, self.parameters()),
                                lr=self.hparams["lr"])

    def predict(self, input):
        y_hat = self(input)
        return y_hat

  
    def training_step(self, batch, batch_idx):
       x = batch[self.hparams["Input_key"]]
        y = batch["label"]
        y_hat = self.predict(x)
        loss = F.cross_entropy(y_hat, y)
        with torch.no_grad():
         y_hat = F.softmax(y_hat, dim=1)
         raw_image = batch[0]
        cam_img = self.cam(x=raw_image[None], class_idx=None)
        cam_img
        gradcam_img = self.gradcam(x=raw_image[None], class_idx=None)
         return {'loss': loss, "train_y_hat": y_hat, "train_y": y}

    def validation_step(self, batch, batch_idx):
        x = batch[self.hparams["Input_key"]]
        y = batch["label"]
        y_hat = self.predict(x)
        loss = F.cross_entropy(y_hat, y)
        with torch.no_grad():
            y_hat = F.softmax(y_hat, dim=1)
        return {'val_loss': loss, "val_y_hat": y_hat, "val_y": y}

    def test_step(self, batch, batch_idx):
         x = batch[self.hparams["Input_key"]]
        y = batch["label"]
        y_hat = self.predict(x)
        loss = F.cross_entropy(y_hat, y)
        with torch.no_grad():
            y_hat = F.softmax(y_hat, dim=1)
        return {'test_loss': loss, "test_y_hat": y_hat, "test_y": y}

if __name__ == "__main__":
 
            my_model = Model(hparams)
           

            trainer.fit(my_model, mri_dataset)

[Nic-Ma]

Hi @wyli ,

Could you please help share some info about GradCam here?

Thanks in advance.

[rijobro]

@roshnikotian15 Can you give info on your error or what's not working as you expect?