MainCode_Approach1.py

# -*- coding: utf-8 -*-
"""MJAhmadi_NNDL_HW2_Q2.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1_46STgIZ3gxXO0BCGQDOU8ZQZkIKbcFj

# Section 2.1

## Load and Unzip Dataset
"""

!pip install --upgrade --no-cache-dir gdown
!gdown 1JwIyR97fXRfaciFjm4NLualTzq2XaDTg

import zipfile
zip_file_path = '/content/archive.zip'
folder_path = '/content/Dataset'
# Extract the zip file to the specified folder
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
    zip_ref.extractall(folder_path)

import os
file_path = "/content/archive.zip"
if os.path.exists(file_path):
    os.remove(file_path)
    print(f"{file_path} has been deleted successfully.")
else:
    print(f"{file_path} does not exist.")

import os

# Define the path to the parent folder
parent_folder_path = "/content/Dataset"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

"""## Load and Unzip Dataset"""

import os

# Define the name of the new folder
new_folder_name = "AllDataPreprocess"

# Define the path where the new folder will be created
parent_folder_path = "/content"

# Check if the new folder already exists
new_folder_path = os.path.join(parent_folder_path, new_folder_name)
if not os.path.exists(new_folder_path):
    # Create the new folder
    os.makedirs(new_folder_path)
    print(f"Created new folder: {new_folder_path}")
else:
    print(f"Folder already exists: {new_folder_path}")

import os

# Define the name of the new folder
new_folder_name = "AllDataPreprocessNormal"

# Define the path where the new folder will be created
parent_folder_path = "/content"

# Check if the new folder already exists
new_folder_path = os.path.join(parent_folder_path, new_folder_name)
if not os.path.exists(new_folder_path):
    # Create the new folder
    os.makedirs(new_folder_path)
    print(f"Created new folder: {new_folder_path}")
else:
    print(f"Folder already exists: {new_folder_path}")

import os

# Define the name of the new folder
new_folder_name = "AllDataPreprocessNormalAug"

# Define the path where the new folder will be created
parent_folder_path = "/content"

# Check if the new folder already exists
new_folder_path = os.path.join(parent_folder_path, new_folder_name)
if not os.path.exists(new_folder_path):
    # Create the new folder
    os.makedirs(new_folder_path)
    print(f"Created new folder: {new_folder_path}")
else:
    print(f"Folder already exists: {new_folder_path}")

import os

# Define the name of the new folder
new_folder_name = "AllDataPreprocessAug"

# Define the path where the new folder will be created
parent_folder_path = "/content"

# Check if the new folder already exists
new_folder_path = os.path.join(parent_folder_path, new_folder_name)
if not os.path.exists(new_folder_path):
    # Create the new folder
    os.makedirs(new_folder_path)
    print(f"Created new folder: {new_folder_path}")
else:
    print(f"Folder already exists: {new_folder_path}")

import os

# Define the name of the new folder
new_folder_name = "FinalDataset"

# Define the path where the new folder will be created
parent_folder_path = "/content"

# Check if the new folder already exists
new_folder_path = os.path.join(parent_folder_path, new_folder_name)
if not os.path.exists(new_folder_path):
    # Create the new folder
    os.makedirs(new_folder_path)
    print(f"Created new folder: {new_folder_path}")
else:
    print(f"Folder already exists: {new_folder_path}")

import os

# Define the name of the new folder
new_folder_name = "FinalDatasetNormal"

# Define the path where the new folder will be created
parent_folder_path = "/content"

# Check if the new folder already exists
new_folder_path = os.path.join(parent_folder_path, new_folder_name)
if not os.path.exists(new_folder_path):
    # Create the new folder
    os.makedirs(new_folder_path)
    print(f"Created new folder: {new_folder_path}")
else:
    print(f"Folder already exists: {new_folder_path}")

"""### Prepare Data"""

import os
import shutil
from torch.utils.data import Dataset, DataLoader
from torchvision.datasets.folder import default_loader

# Set the path to your input folder here
input_folder_path = "/content/Dataset/chest_xray/chest_xray"

# Define the classes in your dataset
classes = ["NORMAL", "PNEUMONIA"]

# Define the path to the output folder where you want to save the combined data
output_folder_path = os.path.join(input_folder_path, "AllData")

# Create the output folder if it doesn't exist
if not os.path.exists(output_folder_path):
    os.makedirs(output_folder_path)

# Loop through the train, test, and val folders
for split in ["train", "test", "val"]:
    split_folder_path = os.path.join(input_folder_path, split)

    # Loop through the NORMAL and PNEUMONIA folders in each split folder
    for class_name in classes:
        class_folder_path = os.path.join(split_folder_path, class_name)

        # Loop through the image files in each class folder and copy them to the output folder
        for file_name in os.listdir(class_folder_path):
            if file_name.endswith(".jpeg"):
                src_path = os.path.join(class_folder_path, file_name)
                dst_path = os.path.join(output_folder_path, class_name, file_name)

                # Create the class folder in the output folder if it doesn't exist
                if not os.path.exists(os.path.join(output_folder_path, class_name)):
                    os.makedirs(os.path.join(output_folder_path, class_name))

                # Copy the image file to the output folder
                shutil.copyfile(src_path, dst_path)

# Define a custom PyTorch dataset to load the combined data
class CustomDataset(Dataset):
    def __init__(self, root, classes, transform=None, loader=default_loader):
        self.root = root
        self.classes = classes
        self.transform = transform
        self.loader = loader
        self.samples = []

        # Loop through the NORMAL and PNEUMONIA classes and their respective image folders in the output folder
        for class_name in classes:
            class_folder_path = os.path.join(root, class_name)
            for file_name in os.listdir(class_folder_path):
                if file_name.endswith(".jpeg"):
                    self.samples.append((os.path.join(class_folder_path, file_name), classes.index(class_name)))

    def __getitem__(self, index):
        path, target = self.samples[index]
        sample = self.loader(path)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample, target

    def __len__(self):
        return len(self.samples)

# Load the combined data using the custom PyTorch dataset
dataset = CustomDataset(output_folder_path, classes)

# Use the PyTorch DataLoader to create batches of data for training/testing
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

import os

# Define the path to the parent folder
parent_folder_path = "/content/Dataset/chest_xray/chest_xray/AllData"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

"""### Just Preprocess"""

import os
import torch
import torchvision
import torchvision.transforms as transforms
from torchvision.datasets import ImageFolder

# Define the path to the input and output folders
input_folder_path = "/content/Dataset/chest_xray/chest_xray/AllData"
output_folder_path = "/content/AllDataPreprocess"

# Define the transformation pipeline to apply to the images
transform = transforms.Compose([
    transforms.Resize((128, 128)),
    transforms.ToTensor()
])

# Load the data from the input folder using the ImageFolder dataset
data = ImageFolder(input_folder_path, transform=transform)

# Create the output folder if it doesn't exist
if not os.path.exists(output_folder_path):
    os.makedirs(output_folder_path)

# Loop through the NORMAL and PNEUMONIA classes and their respective image folders in the input folder
for class_name in data.classes:
    class_folder_path = os.path.join(output_folder_path, class_name)
    if not os.path.exists(class_folder_path):
        os.makedirs(class_folder_path)

    # Loop through the images in the class folder, apply the transformation pipeline, and save the new images to the output folder
    for i in range(len(data)):
        if data.targets[i] == data.class_to_idx[class_name]:
            image, _ = data[i]
            file_name = data.samples[i][0].split("/")[-1]
            output_path = os.path.join(class_folder_path, file_name)
            torchvision.utils.save_image(image, output_path)

import os
import numpy as np
import torch
import torchvision.transforms as transforms
from PIL import Image

# Define the path to the folder containing the images
folder_path = "/content/AllDataPreprocess"

# Define the transforms to be applied to each image
transform = transforms.Compose([
    transforms.Resize((128, 128)),
    transforms.ToTensor()
])

# Define the accumulator variables for the mean and standard deviation
normal_images = []
pneumonia_images = []

# Loop through all files in the folder
for folder_name in ["NORMAL", "PNEUMONIA"]:
    folder = os.path.join(folder_path, folder_name)
    for file_name in os.listdir(folder):
        # Check if the file is an image (JPEG or PNG)
        if file_name.endswith(".jpg") or file_name.endswith(".jpeg") or file_name.endswith(".png"):
            # Open the image file and apply the transforms
            image_path = os.path.join(folder, file_name)
            image = Image.open(image_path)
            image = transform(image)

            # Add the image to the corresponding accumulator variable
            if folder_name == "NORMAL":
                normal_images.append(image)
            else:
                pneumonia_images.append(image)

# Concatenate all images and calculate the mean and standard deviation
normal_images = torch.stack(normal_images)
pneumonia_images = torch.stack(pneumonia_images)
all_images = torch.cat([normal_images, pneumonia_images], dim=0)
mean = torch.mean(all_images, dim=(0, 2, 3))
std = torch.std(all_images, dim=(0, 2, 3))

# Output the mean and standard deviation
print(f"mean={mean.tolist()}, std={std.tolist()}")

"""### Preprocess + Normal"""

import os
import torch
import torchvision
import torchvision.transforms as transforms
from torchvision.datasets import ImageFolder

# Define the path to the input and output folders
input_folder_path = "/content/AllDataPreprocess"
output_folder_path = "/content/AllDataPreprocessNormal"

# Define the transformation pipeline to apply to the images
transform = transforms.Compose([
    transforms.Resize((128, 128)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.4815806746482849, 0.4815806746482849, 0.4815806746482849], std=[0.23505905270576477, 0.23505905270576477, 0.23505905270576477])
])

# Load the data from the input folder using the ImageFolder dataset
data = ImageFolder(input_folder_path, transform=transform)

# Create the output folder if it doesn't exist
if not os.path.exists(output_folder_path):
    os.makedirs(output_folder_path)

# Loop through the NORMAL and PNEUMONIA classes and their respective image folders in the input folder
for class_name in data.classes:
    class_folder_path = os.path.join(output_folder_path, class_name)
    if not os.path.exists(class_folder_path):
        os.makedirs(class_folder_path)

    # Loop through the images in the class folder, apply the transformation pipeline, and save the new images to the output folder
    for i in range(len(data)):
        if data.targets[i] == data.class_to_idx[class_name]:
            image, _ = data[i]
            file_name = data.samples[i][0].split("/")[-1]
            output_path = os.path.join(class_folder_path, file_name)
            torchvision.utils.save_image(image, output_path)

import os

# Define the path to the parent folder
parent_folder_path = "/content/AllDataPreprocessNormal"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

"""### Preprocess + Normal + AUG



"""

import os
import torch
import torchvision
import torchvision.transforms as transforms
from torchvision.datasets import ImageFolder

# Define the path to the input and output folders
input_folder_path = "/content/Dataset/chest_xray/chest_xray/AllData"
output_folder_path = "/content/AllDataPreprocessNormalAug"

# Define the transformation pipeline to apply to the images
transform = transforms.Compose([
    transforms.Resize((128, 128)),
    transforms.RandomAffine(degrees=30, shear=0.2, scale=(0.8, 1.2), translate=(0.2, 0.2)),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    # transforms.Normalize(mean=[0.0, 0.0, 0.0], std=[1.0/255, 1.0/255, 1.0/255])
    transforms.Normalize(mean=[0.4815806746482849, 0.4815806746482849, 0.4815806746482849], std=[0.23505905270576477, 0.23505905270576477, 0.23505905270576477])
])

# Load the data from the input folder using the ImageFolder dataset
data = ImageFolder(input_folder_path, transform=transform)

# Create the output folder if it doesn't exist
if not os.path.exists(output_folder_path):
    os.makedirs(output_folder_path)

# Loop through the NORMAL and PNEUMONIA classes and their respective image folders in the input folder
for class_name in data.classes:
    class_folder_path = os.path.join(output_folder_path, class_name)
    if not os.path.exists(class_folder_path):
        os.makedirs(class_folder_path)

    # Loop through the images in the class folder, apply the transformation pipeline, and save the new images to the output folder
    for i in range(len(data)):
        if data.targets[i] == data.class_to_idx[class_name]:
            image, _ = data[i]
            file_name = data.samples[i][0].split("/")[-1]
            output_path = os.path.join(class_folder_path, file_name)
            torchvision.utils.save_image(image, output_path)

import os

# Define the path to the parent folder
parent_folder_path = "/content/AllDataPreprocessNormalAug"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

"""### Preprocess + AUG



"""

import os
import torch
import torchvision
import torchvision.transforms as transforms
from torchvision.datasets import ImageFolder

# Define the path to the input and output folders
input_folder_path = "/content/Dataset/chest_xray/chest_xray/AllData"
output_folder_path = "/content/AllDataPreprocessAug"

# Define the transformation pipeline to apply to the images
transform = transforms.Compose([
    transforms.Resize((128, 128)),
    transforms.RandomAffine(degrees=30, shear=0.2, scale=(0.8, 1.2), translate=(0.2, 0.2)),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor()
    # transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    # transforms.Normalize(mean=[0.4815806746482849, 0.4815806746482849, 0.4815806746482849], std=[0.23505905270576477, 0.23505905270576477, 0.23505905270576477])
])

# Load the data from the input folder using the ImageFolder dataset
data = ImageFolder(input_folder_path, transform=transform)

# Create the output folder if it doesn't exist
if not os.path.exists(output_folder_path):
    os.makedirs(output_folder_path)

# Loop through the NORMAL and PNEUMONIA classes and their respective image folders in the input folder
for class_name in data.classes:
    class_folder_path = os.path.join(output_folder_path, class_name)
    if not os.path.exists(class_folder_path):
        os.makedirs(class_folder_path)

    # Loop through the images in the class folder, apply the transformation pipeline, and save the new images to the output folder
    for i in range(len(data)):
        if data.targets[i] == data.class_to_idx[class_name]:
            image, _ = data[i]
            file_name = data.samples[i][0].split("/")[-1]
            output_path = os.path.join(class_folder_path, file_name)
            torchvision.utils.save_image(image, output_path)

import os

# Define the path to the parent folder
parent_folder_path = "/content/AllDataPreprocessAug"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

"""### Merge and make Final Dataset"""

import shutil

# replace `folder_to_delete` with the name of the folder you want to delete
shutil.rmtree("/content/FinalDatasetTVT")

import os
import shutil

# Path to the new folder
new_folder_path = "/content/FinalDataset"

# Path to the first and second folders
first_folder_path = '/content/AllDataPreprocess'
second_folder_path = '/content/AllDataPreprocessAug'

# Create the new subfolders
os.makedirs(os.path.join(new_folder_path, "NORMAL"))
os.makedirs(os.path.join(new_folder_path, "PNEUMONIA"))

# Copy the files from the first folder
for filename in os.listdir(os.path.join(first_folder_path, "NORMAL")):
    new_filename = filename[:-5] + "_1" + filename[-5:]
    shutil.copy(os.path.join(first_folder_path, "NORMAL", filename), 
                os.path.join(new_folder_path, "NORMAL", new_filename))
for filename in os.listdir(os.path.join(first_folder_path, "PNEUMONIA")):
    new_filename = filename[:-5] + "_1" + filename[-5:]
    shutil.copy(os.path.join(first_folder_path, "PNEUMONIA", filename), 
                os.path.join(new_folder_path, "PNEUMONIA", new_filename))

# Copy the files from the second folder
for filename in os.listdir(os.path.join(second_folder_path, "NORMAL")):
    new_filename = filename[:-5] + "_2" + filename[-5:]
    shutil.copy(os.path.join(second_folder_path, "NORMAL", filename), 
                os.path.join(new_folder_path, "NORMAL", new_filename))
for filename in os.listdir(os.path.join(second_folder_path, "PNEUMONIA")):
    new_filename = filename[:-5] + "_2" + filename[-5:]
    shutil.copy(os.path.join(second_folder_path, "PNEUMONIA", filename), 
                os.path.join(new_folder_path, "PNEUMONIA", new_filename))

import os

# Define the path to the parent folder
parent_folder_path = "/content/FinalDataset"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

import os
import shutil

# Path to the new folder
new_folder_path = "/content/FinalDatasetNormal"

# Path to the first and second folders
first_folder_path = '/content/AllDataPreprocessNormal'
second_folder_path = '/content/AllDataPreprocessNormalAug'

# Create the new subfolders
os.makedirs(os.path.join(new_folder_path, "NORMAL"))
os.makedirs(os.path.join(new_folder_path, "PNEUMONIA"))

# Copy the files from the first folder
for filename in os.listdir(os.path.join(first_folder_path, "NORMAL")):
    new_filename = filename[:-5] + "_1" + filename[-5:]
    shutil.copy(os.path.join(first_folder_path, "NORMAL", filename), 
                os.path.join(new_folder_path, "NORMAL", new_filename))
for filename in os.listdir(os.path.join(first_folder_path, "PNEUMONIA")):
    new_filename = filename[:-5] + "_1" + filename[-5:]
    shutil.copy(os.path.join(first_folder_path, "PNEUMONIA", filename), 
                os.path.join(new_folder_path, "PNEUMONIA", new_filename))

# Copy the files from the second folder
for filename in os.listdir(os.path.join(second_folder_path, "NORMAL")):
    new_filename = filename[:-5] + "_2" + filename[-5:]
    shutil.copy(os.path.join(second_folder_path, "NORMAL", filename), 
                os.path.join(new_folder_path, "NORMAL", new_filename))
for filename in os.listdir(os.path.join(second_folder_path, "PNEUMONIA")):
    new_filename = filename[:-5] + "_2" + filename[-5:]
    shutil.copy(os.path.join(second_folder_path, "PNEUMONIA", filename), 
                os.path.join(new_folder_path, "PNEUMONIA", new_filename))

import os

# Define the path to the parent folder
parent_folder_path = "/content/FinalDatasetNormal"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

"""### Train,Valid,Test..."""

import os
import shutil
import random

# Set the path to the "FinalDataset" folder
data_dir = "/content/FinalDataset"

# Set the path to the output directory
output_dir = "/content/FinalDatasetTVT"

# Set the train/validation/test split ratios
train_ratio = 0.6
val_ratio = 0.2
test_ratio = 0.2

# Create the output directories
os.makedirs(os.path.join(output_dir, "train", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "train", "PNEUMONIA"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "val", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "val", "PNEUMONIA"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "test", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "test", "PNEUMONIA"), exist_ok=True)

# Get the list of image files in each class folder
normal_files = os.listdir(os.path.join(data_dir, "NORMAL"))
pneumonia_files = os.listdir(os.path.join(data_dir, "PNEUMONIA"))

# Shuffle the lists to randomize the order
random.shuffle(normal_files)
random.shuffle(pneumonia_files)

# Calculate the number of images for each split
num_normal = len(normal_files)
num_pneumonia = len(pneumonia_files)
num_train_normal = int(num_normal * train_ratio)
num_train_pneumonia = int(num_pneumonia * train_ratio)
num_val_normal = int(num_normal * val_ratio)
num_val_pneumonia = int(num_pneumonia * val_ratio)
num_test_normal = int(num_normal * test_ratio)
num_test_pneumonia = int(num_pneumonia * test_ratio)

# Copy the image files to the output directories for each split
for i, file in enumerate(normal_files):
    if i < num_train_normal:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "train", "NORMAL"))
    elif i < num_train_normal + num_val_normal:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "val", "NORMAL"))
    else:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "test", "NORMAL"))
for i, file in enumerate(pneumonia_files):
    if i < num_train_pneumonia:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "train", "PNEUMONIA"))
    elif i < num_train_pneumonia + num_val_pneumonia:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "val", "PNEUMONIA"))
    else:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "test", "PNEUMONIA"))

import os
import shutil
import random

# Set the path to the "FinalDataset" folder
data_dir = "/content/FinalDatasetNormal"

# Set the path to the output directory
output_dir = "/content/FinalDatasetNormalTVT"

# Set the train/validation/test split ratios
train_ratio = 0.6
val_ratio = 0.2
test_ratio = 0.2

# Create the output directories
os.makedirs(os.path.join(output_dir, "train", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "train", "PNEUMONIA"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "val", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "val", "PNEUMONIA"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "test", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "test", "PNEUMONIA"), exist_ok=True)

# Get the list of image files in each class folder
normal_files = os.listdir(os.path.join(data_dir, "NORMAL"))
pneumonia_files = os.listdir(os.path.join(data_dir, "PNEUMONIA"))

# Shuffle the lists to randomize the order
random.shuffle(normal_files)
random.shuffle(pneumonia_files)

# Calculate the number of images for each split
num_normal = len(normal_files)
num_pneumonia = len(pneumonia_files)
num_train_normal = int(num_normal * train_ratio)
num_train_pneumonia = int(num_pneumonia * train_ratio)
num_val_normal = int(num_normal * val_ratio)
num_val_pneumonia = int(num_pneumonia * val_ratio)
num_test_normal = int(num_normal * test_ratio)
num_test_pneumonia = int(num_pneumonia * test_ratio)

# Copy the image files to the output directories for each split
for i, file in enumerate(normal_files):
    if i < num_train_normal:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "train", "NORMAL"))
    elif i < num_train_normal + num_val_normal:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "val", "NORMAL"))
    else:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "test", "NORMAL"))
for i, file in enumerate(pneumonia_files):
    if i < num_train_pneumonia:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "train", "PNEUMONIA"))
    elif i < num_train_pneumonia + num_val_pneumonia:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "val", "PNEUMONIA"))
    else:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "test", "PNEUMONIA"))

import os

# Define the path to the parent folder
parent_folder_path = "/content/FinalDatasetNormalTVT"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

"""# Section 2.2

## Pre
"""

!pip install --upgrade --no-cache-dir gdown
!gdown 1JwIyR97fXRfaciFjm4NLualTzq2XaDTg

import zipfile
zip_file_path = '/content/archive.zip'
folder_path = '/content/Dataset'
# Extract the zip file to the specified folder
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
    zip_ref.extractall(folder_path)

import os
file_path = "/content/archive.zip"
if os.path.exists(file_path):
    os.remove(file_path)
    print(f"{file_path} has been deleted successfully.")
else:
    print(f"{file_path} does not exist.")

import os

# Define the path to the parent folder
parent_folder_path = "/content/Dataset"

# Recursively iterate through all subfolders and files in the parent folder
for dirpath, dirnames, filenames in os.walk(parent_folder_path):
    # Get the number of JPEG files in the current directory
    jpeg_count = sum(1 for filename in filenames if filename.lower().endswith('.jpeg'))
    
    # Print out the results for the current directory
    if jpeg_count > 0:
        # Get the relative path to the current directory
        relative_path = os.path.relpath(dirpath, parent_folder_path)
        
        # Print out the results
        print(f"{jpeg_count} (.jpeg) files are in {os.path.join(parent_folder_path, relative_path)}")

import os
import shutil
from torch.utils.data import Dataset, DataLoader
from torchvision.datasets.folder import default_loader

# Set the path to your input folder here
input_folder_path = "/content/Dataset/chest_xray/chest_xray"

# Define the classes in your dataset
classes = ["NORMAL", "PNEUMONIA"]

# Define the path to the output folder where you want to save the combined data
output_folder_path = os.path.join(input_folder_path, "AllData")

# Create the output folder if it doesn't exist
if not os.path.exists(output_folder_path):
    os.makedirs(output_folder_path)

# Loop through the train, test, and val folders
for split in ["train", "test", "val"]:
    split_folder_path = os.path.join(input_folder_path, split)

    # Loop through the NORMAL and PNEUMONIA folders in each split folder
    for class_name in classes:
        class_folder_path = os.path.join(split_folder_path, class_name)

        # Loop through the image files in each class folder and copy them to the output folder
        for file_name in os.listdir(class_folder_path):
            if file_name.endswith(".jpeg"):
                src_path = os.path.join(class_folder_path, file_name)
                dst_path = os.path.join(output_folder_path, class_name, file_name)

                # Create the class folder in the output folder if it doesn't exist
                if not os.path.exists(os.path.join(output_folder_path, class_name)):
                    os.makedirs(os.path.join(output_folder_path, class_name))

                # Copy the image file to the output folder
                shutil.copyfile(src_path, dst_path)

# Define a custom PyTorch dataset to load the combined data
class CustomDataset(Dataset):
    def __init__(self, root, classes, transform=None, loader=default_loader):
        self.root = root
        self.classes = classes
        self.transform = transform
        self.loader = loader
        self.samples = []

        # Loop through the NORMAL and PNEUMONIA classes and their respective image folders in the output folder
        for class_name in classes:
            class_folder_path = os.path.join(root, class_name)
            for file_name in os.listdir(class_folder_path):
                if file_name.endswith(".jpeg"):
                    self.samples.append((os.path.join(class_folder_path, file_name), classes.index(class_name)))

    def __getitem__(self, index):
        path, target = self.samples[index]
        sample = self.loader(path)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample, target

    def __len__(self):
        return len(self.samples)

# Load the combined data using the custom PyTorch dataset
dataset = CustomDataset(output_folder_path, classes)

# Use the PyTorch DataLoader to create batches of data for training/testing
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

import os
import shutil
import random

# Set the path to the "FinalDataset" folder
data_dir = "/content/Dataset/chest_xray/chest_xray/AllData"

# Set the path to the output directory
output_dir = "/content/FinalDatasetTVT2"

# Set the train/validation/test split ratios
train_ratio = 0.6
val_ratio = 0.2
test_ratio = 0.2

# Create the output directories
os.makedirs(os.path.join(output_dir, "train", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "train", "PNEUMONIA"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "val", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "val", "PNEUMONIA"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "test", "NORMAL"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "test", "PNEUMONIA"), exist_ok=True)

# Get the list of image files in each class folder
normal_files = os.listdir(os.path.join(data_dir, "NORMAL"))
pneumonia_files = os.listdir(os.path.join(data_dir, "PNEUMONIA"))

# Shuffle the lists to randomize the order
random.shuffle(normal_files)
random.shuffle(pneumonia_files)

# Calculate the number of images for each split
num_normal = len(normal_files)
num_pneumonia = len(pneumonia_files)
num_train_normal = int(num_normal * train_ratio)
num_train_pneumonia = int(num_pneumonia * train_ratio)
num_val_normal = int(num_normal * val_ratio)
num_val_pneumonia = int(num_pneumonia * val_ratio)
num_test_normal = int(num_normal * test_ratio)
num_test_pneumonia = int(num_pneumonia * test_ratio)

# Copy the image files to the output directories for each split
for i, file in enumerate(normal_files):
    if i < num_train_normal:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "train", "NORMAL"))
    elif i < num_train_normal + num_val_normal:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "val", "NORMAL"))
    else:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "test", "NORMAL"))
for i, file in enumerate(pneumonia_files):
    if i < num_train_pneumonia:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "train", "PNEUMONIA"))
    elif i < num_train_pneumonia + num_val_pneumonia:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "val", "PNEUMONIA"))
    else:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "test", "PNEUMONIA"))

import os
import shutil
import random

# Set the path to the "FinalDataset" folder
data_dir = "/content/Dataset/chest_xray/chest_xray/AllData"

# Set the path to the output directory
output_dir = "/content/FinalDatasetTVT4"

# Set the train/validation/test split ratios
train_ratio = 0.6
val_ratio = 0.2
test_ratio = 0.2

# Create the output directories
os.makedirs(os.path.join(output_dir, "train", "0"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "train", "1"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "val", "0"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "val", "1"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "test", "0"), exist_ok=True)
os.makedirs(os.path.join(output_dir, "test", "1"), exist_ok=True)

# Get the list of image files in each class folder
normal_files = os.listdir(os.path.join(data_dir, "NORMAL"))
pneumonia_files = os.listdir(os.path.join(data_dir, "PNEUMONIA"))

# Shuffle the lists to randomize the order
random.shuffle(normal_files)
random.shuffle(pneumonia_files)

# Calculate the number of images for each split
num_normal = len(normal_files)
num_pneumonia = len(pneumonia_files)
num_train_normal = int(num_normal * train_ratio)
num_train_pneumonia = int(num_pneumonia * train_ratio)
num_val_normal = int(num_normal * val_ratio)
num_val_pneumonia = int(num_pneumonia * val_ratio)
num_test_normal = int(num_normal * test_ratio)
num_test_pneumonia = int(num_pneumonia * test_ratio)

# Copy the image files to the output directories for each split
for i, file in enumerate(normal_files):
    if i < num_train_normal:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "train", "0"))
    elif i < num_train_normal + num_val_normal:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "val", "0"))
    else:
        shutil.copy(os.path.join(data_dir, "NORMAL", file), os.path.join(output_dir, "test", "0"))
for i, file in enumerate(pneumonia_files):
    if i < num_train_pneumonia:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "train", "1"))
    elif i < num_train_pneumonia + num_val_pneumonia:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "val", "1"))
    else:
        shutil.copy(os.path.join(data_dir, "PNEUMONIA", file), os.path.join(output_dir, "test", "1"))

from PIL import Image
import os

# Set the path to the directory containing the images
directory = '/content/FinalDatasetTVT4/'

# Loop through all the subdirectories and files in the directory
for root, dirs, files in os.walk(directory):
    for filename in files:
        if filename.endswith('.jpg') or filename.endswith('.jpeg') or filename.endswith('.png'):
            # Open the image and convert it to RGB format
            img = Image.open(os.path.join(root, filename)).convert('RGB')
            
            # Save the image back to the same file
            img.save(os.path.join(root, filename))

import tensorflow as tf
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications import EfficientNetB2
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications import EfficientNetB0, EfficientNetB2
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from keras.layers import GlobalAveragePooling2D
from sklearn.metrics import precision_recall_curve
import seaborn as sns

!pip install efficientnet

"""## Imp1"""

import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout
from tensorflow.keras.models import Model
from efficientnet.tfkeras import EfficientNetB2
import numpy as np
from sklearn.metrics import roc_auc_score, recall_score, f1_score, precision_score

train_datagen = ImageDataGenerator(
    rescale=1.0/255,
    shear_range=0.2,
    width_shift_range=0.2,
    height_shift_range=0.2,
    rotation_range=30,
    horizontal_flip=True,
    zoom_range=0.2
)

val_datagen = ImageDataGenerator(rescale=1.0/255)
test_datagen = ImageDataGenerator(rescale=1.0/255)

train_generator = train_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/train',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

val_generator = val_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/val',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

test_generator = test_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/test',
    target_size=(128, 128),
    batch_size=1174,
    class_mode='binary')

# separate data and labels for training set
x_train, y_train = train_generator.next()

# separate data and labels for validation set
x_val, y_val = val_generator.next()

# separate data and labels for test set
x_test, y_test = test_generator.next()

# load pre-trained model
base_model = EfficientNetB2(weights='imagenet', include_top=False, input_shape=(128, 128, 3))

# add new layers
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(64, activation='relu')(x)
x = Dropout(0.2)(x)
predictions = Dense(1, activation='sigmoid')(x)

model = Model(inputs=base_model.input, outputs=predictions)

# freeze pre-trained layers
for layer in base_model.layers:
    layer.trainable = False

import numpy as np
from sklearn.utils.class_weight import compute_class_weight

# class_labels = np.unique(y_train)
class_weights = compute_class_weight(
                                        class_weight = "balanced",
                                        classes = np.unique(y_train),
                                        y = y_train                                                    
                                    )
class_weights = dict(zip(np.unique(y_train), class_weights))

# define callbacks
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping

# save the model weights after each epoch if the validation loss decreased
checkpoint = ModelCheckpoint('best_model.h5', 
                             save_best_only=True, 
                             save_weights_only=True,
                             monitor='val_loss', 
                             mode='min', verbose=1)

# reduce learning rate when the validation loss plateaus
reduce_lr = ReduceLROnPlateau(monitor='val_loss', 
                              factor=0.2, 
                              patience=10, 
                              min_lr=0.0001, verbose=1)

# stop training if the validation loss doesn't improve for 30 consecutive epochs
early_stop = EarlyStopping(monitor='val_loss', patience=10)

# compile the model
model.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy'])

# train the model
history = model.fit(x_train, y_train,
                    validation_data=(x_val, y_val),
                    epochs=30,
                    batch_size=128,
                    callbacks=[checkpoint, reduce_lr, early_stop],
                    class_weight=class_weights
                    )

# evaluate the model on the test set
loss, accuracy = model.evaluate(x_test, y_test, batch_size=128)

print('Test loss:', loss)
print('Test accuracy:', accuracy)

# evaluate the model on the test set
y_pred = model.predict(x_test)
y_pred_classes = np.round(y_pred)
auc_score = roc_auc_score(y_test, y_pred)
recall = recall_score(y_test, y_pred_classes)
f1 = f1_score(y_test, y_pred_classes)
precision = precision_score(y_test, y_pred_classes)

print('Test AUC:', auc_score)
print('Test Recall:', recall)
print('Test F1-score:', f1)
print('Test Precision:', precision)

# plot confusion matrix
cm = confusion_matrix(y_test, y_pred_classes)
sns.heatmap(cm, annot=True, fmt='g')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.savefig('A5.pdf')
plt.show()

# plot PR curve
precision, recall, _ = precision_recall_curve(y_test, y_pred)
plt.plot(recall, precision)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.savefig('A4.pdf')
plt.show()

# plot ROC curve
fpr, tpr, _ = roc_curve(y_test, y_pred)
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.savefig('A3.pdf')
plt.show()

# plot accuracy and loss curves for training and validation sets
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.savefig('A2.pdf')
plt.show()

plt.plot(history.history['loss'], label='Training Loss')
plt.plot(history.history['val_loss'], label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.savefig('A1.pdf')
plt.show()

model.save('my_model.h5')

"""## Imp2"""

import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout
from tensorflow.keras.models import Model
from efficientnet.tfkeras import EfficientNetB2
import numpy as np
from sklearn.metrics import roc_auc_score, recall_score, f1_score, precision_score

train_datagen = ImageDataGenerator(
    rescale=1.0/255,
    shear_range=0.2,
    width_shift_range=0.2,
    height_shift_range=0.2,
    rotation_range=30,
    horizontal_flip=True,
    zoom_range=0.2
)

val_datagen = ImageDataGenerator(rescale=1.0/255)
test_datagen = ImageDataGenerator(rescale=1.0/255)

train_generator = train_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/train',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

val_generator = val_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/val',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

test_generator = test_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/test',
    target_size=(128, 128),
    batch_size=1174,
    class_mode='binary')

# separate data and labels for training set
x_train, y_train = train_generator.next()

# separate data and labels for validation set
x_val, y_val = val_generator.next()

# separate data and labels for test set
x_test, y_test = test_generator.next()

# load pre-trained model
base_model = EfficientNetB2(weights='imagenet', include_top=False, input_shape=(128, 128, 3))

# add new layers
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(64, activation='relu')(x)
x = Dropout(0.2)(x)
predictions = Dense(1, activation='sigmoid')(x)

model = Model(inputs=base_model.input, outputs=predictions)

# freeze pre-trained layers
for layer in base_model.layers:
    layer.trainable = False

import numpy as np
from sklearn.utils.class_weight import compute_class_weight

# class_labels = np.unique(y_train)
class_weights = compute_class_weight(
                                        class_weight = "balanced",
                                        classes = np.unique(y_train),
                                        y = y_train                                                    
                                    )
class_weights = dict(zip(np.unique(y_train), class_weights))

# define callbacks
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping

# save the model weights after each epoch if the validation loss decreased
checkpoint = ModelCheckpoint('best_model.h5', 
                             save_best_only=True, 
                             save_weights_only=True,
                             monitor='val_loss', 
                             mode='min', verbose=1)

# reduce learning rate when the validation loss plateaus
reduce_lr = ReduceLROnPlateau(monitor='val_loss', 
                              factor=0.2, 
                              patience=15, 
                              min_lr=0.0001, verbose=1)

# stop training if the validation loss doesn't improve for 30 consecutive epochs
early_stop = EarlyStopping(monitor='val_loss', patience=20)

# compile the model
model.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy'])

# train the model
history = model.fit(x_train, y_train,
                    validation_data=(x_val, y_val),
                    epochs=50,
                    batch_size=128,
                    callbacks=[checkpoint, reduce_lr, early_stop],
                    class_weight=class_weights
                    )

# evaluate the model on the test set
loss, accuracy = model.evaluate(x_test, y_test, batch_size=1174)

print('Test loss:', loss)
print('Test accuracy:', accuracy)

# evaluate the model on the test set
y_pred = model.predict(x_test)
y_pred_classes = np.round(y_pred)
auc_score = roc_auc_score(y_test, y_pred)
recall = recall_score(y_test, y_pred_classes)
f1 = f1_score(y_test, y_pred_classes)
precision = precision_score(y_test, y_pred_classes)

print('Test AUC:', auc_score)
print('Test Recall:', recall)
print('Test F1-score:', f1)
print('Test Precision:', precision)

# plot confusion matrix
cm = confusion_matrix(y_test, y_pred_classes)
sns.heatmap(cm, annot=True, fmt='g')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.savefig('A5.pdf')
plt.show()

# plot PR curve
precision, recall, _ = precision_recall_curve(y_test, y_pred)
plt.plot(recall, precision)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.savefig('A4.pdf')
plt.show()

# plot ROC curve
fpr, tpr, _ = roc_curve(y_test, y_pred)
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.savefig('A3.pdf')
plt.show()

# plot accuracy and loss curves for training and validation sets
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.savefig('A2.pdf')
plt.show()

plt.plot(history.history['loss'], label='Training Loss')
plt.plot(history.history['val_loss'], label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.savefig('A1.pdf')
plt.show()

model.save('my_model.h5')

"""## Imp3"""

import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout
from tensorflow.keras.models import Model
from efficientnet.tfkeras import EfficientNetB2
import numpy as np
from sklearn.metrics import roc_auc_score, recall_score, f1_score, precision_score

train_datagen = ImageDataGenerator(
    rescale=1.0/255,
    shear_range=0.2,
    width_shift_range=0.2,
    height_shift_range=0.2,
    rotation_range=30,
    horizontal_flip=True,
    zoom_range=0.2
)

val_datagen = ImageDataGenerator(rescale=1.0/255)
test_datagen = ImageDataGenerator(rescale=1.0/255)

train_generator = train_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/train',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

val_generator = val_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/val',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

test_generator = test_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/test',
    target_size=(128, 128),
    batch_size=1174,
    class_mode='binary')

# separate data and labels for training set
x_train, y_train = train_generator.next()

# separate data and labels for validation set
x_val, y_val = val_generator.next()

# separate data and labels for test set
x_test, y_test = test_generator.next()

# load pre-trained model
base_model = EfficientNetB2(weights='imagenet', include_top=False, input_shape=(128, 128, 3))

# add new layers
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(64, activation='relu')(x)
x = Dropout(0.2)(x)
predictions = Dense(1, activation='sigmoid')(x)

model = Model(inputs=base_model.input, outputs=predictions)

# freeze pre-trained layers
for layer in base_model.layers:
    layer.trainable = True

import numpy as np
from sklearn.utils.class_weight import compute_class_weight

# class_labels = np.unique(y_train)
class_weights = compute_class_weight(
                                        class_weight = "balanced",
                                        classes = np.unique(y_train),
                                        y = y_train                                                    
                                    )
class_weights = dict(zip(np.unique(y_train), class_weights))

# define callbacks
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping

# save the model weights after each epoch if the validation loss decreased
checkpoint = ModelCheckpoint('best_model.h5', 
                             save_best_only=True, 
                             save_weights_only=True,
                             monitor='val_loss', 
                             mode='min', verbose=1)

# reduce learning rate when the validation loss plateaus
reduce_lr = ReduceLROnPlateau(monitor='val_loss', 
                              factor=0.2, 
                              patience=10, 
                              min_lr=0.0001, verbose=1)

# stop training if the validation loss doesn't improve for 30 consecutive epochs
early_stop = EarlyStopping(monitor='val_loss', patience=10)

# compile the model
model.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy'])

# train the model
history = model.fit(x_train, y_train,
                    validation_data=(x_val, y_val),
                    epochs=11,
                    batch_size=128,
                    callbacks=[checkpoint, reduce_lr, early_stop],
                    class_weight=class_weights
                    )

# evaluate the model on the test set
loss, accuracy = model.evaluate(x_test, y_test, batch_size=1174)

print('Test loss:', loss)
print('Test accuracy:', accuracy)

# evaluate the model on the test set
y_pred = model.predict(x_test)
y_pred_classes = np.round(y_pred)
auc_score = roc_auc_score(y_test, y_pred)
recall = recall_score(y_test, y_pred_classes)
f1 = f1_score(y_test, y_pred_classes)
precision = precision_score(y_test, y_pred_classes)

print('Test AUC:', auc_score)
print('Test Recall:', recall)
print('Test F1-score:', f1)
print('Test Precision:', precision)

# plot confusion matrix
cm = confusion_matrix(y_test, y_pred_classes)
sns.heatmap(cm, annot=True, fmt='g')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.savefig('A5.pdf')
plt.show()

# plot PR curve
precision, recall, _ = precision_recall_curve(y_test, y_pred)
plt.plot(recall, precision)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.savefig('A4.pdf')
plt.show()

# plot ROC curve
fpr, tpr, _ = roc_curve(y_test, y_pred)
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.savefig('A3.pdf')
plt.show()

# plot accuracy and loss curves for training and validation sets
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.savefig('A2.pdf')
plt.show()

plt.plot(history.history['loss'], label='Training Loss')
plt.plot(history.history['val_loss'], label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.savefig('A1.pdf')
plt.show()

model.save('my_model.h5')

import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout
from tensorflow.keras.models import Model
from efficientnet.tfkeras import EfficientNetB2
import numpy as np
from sklearn.metrics import roc_auc_score, recall_score, f1_score, precision_score

train_datagen = ImageDataGenerator(
    rescale=1.0/255,
    shear_range=0.2,
    width_shift_range=0.2,
    height_shift_range=0.2,
    rotation_range=30,
    horizontal_flip=True,
    zoom_range=0.2
)

val_datagen = ImageDataGenerator(rescale=1.0/255)
test_datagen = ImageDataGenerator(rescale=1.0/255)

train_generator = train_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/train',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

val_generator = val_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/val',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

test_generator = test_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/test',
    target_size=(128, 128),
    batch_size=1174,
    class_mode='binary')

# separate data and labels for training set
x_train, y_train = train_generator.next()

# separate data and labels for validation set
x_val, y_val = val_generator.next()

# separate data and labels for test set
x_test, y_test = test_generator.next()

# load pre-trained model
base_model = EfficientNetB2(weights='imagenet', include_top=False, input_shape=(128, 128, 3))

# add new layers
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(64, activation='relu')(x)
x = Dropout(0.2)(x)
predictions = Dense(1, activation='sigmoid')(x)

model = Model(inputs=base_model.input, outputs=predictions)

# freeze pre-trained layers
for layer in base_model.layers:
    layer.trainable = True

import numpy as np
from sklearn.utils.class_weight import compute_class_weight

# class_labels = np.unique(y_train)
class_weights = compute_class_weight(
                                        class_weight = "balanced",
                                        classes = np.unique(y_train),
                                        y = y_train                                                    
                                    )
class_weights = dict(zip(np.unique(y_train), class_weights))

# define callbacks
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping

# save the model weights after each epoch if the validation loss decreased
checkpoint = ModelCheckpoint('best_model.h5', 
                             save_best_only=True, 
                             save_weights_only=True,
                             monitor='val_loss', 
                             mode='min', verbose=1)

# reduce learning rate when the validation loss plateaus
reduce_lr = ReduceLROnPlateau(monitor='val_loss', 
                              factor=0.2, 
                              patience=10, 
                              min_lr=0.0001, verbose=1)

# stop training if the validation loss doesn't improve for 30 consecutive epochs
early_stop = EarlyStopping(monitor='val_loss', patience=10)

# compile the model
model.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy'])

# train the model
history = model.fit(x_train, y_train,
                    validation_data=(x_val, y_val),
                    epochs=11,
                    batch_size=128,
                    callbacks=[checkpoint, reduce_lr, early_stop],
                    class_weight=class_weights
                    )

# evaluate the model on the test set
loss, accuracy = model.evaluate(x_test, y_test, batch_size=1174)

print('Test loss:', loss)
print('Test accuracy:', accuracy)

# evaluate the model on the test set
y_pred = model.predict(x_test)
y_pred_classes = np.round(y_pred)
auc_score = roc_auc_score(y_test, y_pred)
recall = recall_score(y_test, y_pred_classes)
f1 = f1_score(y_test, y_pred_classes)
precision = precision_score(y_test, y_pred_classes)

print('Test AUC:', auc_score)
print('Test Recall:', recall)
print('Test F1-score:', f1)
print('Test Precision:', precision)

# plot confusion matrix
cm = confusion_matrix(y_test, y_pred_classes)
sns.heatmap(cm, annot=True, fmt='g')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.savefig('A5.pdf')
plt.show()

# plot PR curve
precision, recall, _ = precision_recall_curve(y_test, y_pred)
plt.plot(recall, precision)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.savefig('A4.pdf')
plt.show()

# plot ROC curve
fpr, tpr, _ = roc_curve(y_test, y_pred)
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.savefig('A3.pdf')
plt.show()

# plot accuracy and loss curves for training and validation sets
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.savefig('A2.pdf')
plt.show()

plt.plot(history.history['loss'], label='Training Loss')
plt.plot(history.history['val_loss'], label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.savefig('A1.pdf')
plt.show()

model.save('my_model.h5')

"""## Imp4"""

import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout
from tensorflow.keras.models import Model
from efficientnet.tfkeras import EfficientNetB2
import numpy as np
from sklearn.metrics import roc_auc_score, recall_score, f1_score, precision_score

train_datagen = ImageDataGenerator(
    rescale=1.0/255,
    shear_range=0.2,
    width_shift_range=0.2,
    height_shift_range=0.2,
    rotation_range=30,
    horizontal_flip=True,
    zoom_range=0.2
)

val_datagen = ImageDataGenerator(rescale=1.0/255)
test_datagen = ImageDataGenerator(rescale=1.0/255)

train_generator = train_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/train',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

val_generator = val_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/val',
    target_size=(128, 128),
    batch_size=128,
    class_mode='binary')

test_generator = test_datagen.flow_from_directory(
    '/content/FinalDatasetTVT2/test',
    target_size=(128, 128),
    batch_size=1174,
    class_mode='binary')

# separate data and labels for training set
x_train, y_train = train_generator.next()

# separate data and labels for validation set
x_val, y_val = val_generator.next()

# separate data and labels for test set
x_test, y_test = test_generator.next()

# load pre-trained model
base_model = EfficientNetB2(weights='imagenet', include_top=False, input_shape=(128, 128, 3))

# add new layers
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(64, activation='relu')(x)
x = Dropout(0.2)(x)
predictions = Dense(1, activation='sigmoid')(x)

model = Model(inputs=base_model.input, outputs=predictions)

# freeze pre-trained layers
for layer in base_model.layers:
    layer.trainable = False

import numpy as np
from sklearn.utils.class_weight import compute_class_weight

# class_labels = np.unique(y_train)
class_weights = compute_class_weight(
                                        class_weight = "balanced",
                                        classes = np.unique(y_train),
                                        y = y_train                                                    
                                    )
class_weights = dict(zip(np.unique(y_train), class_weights))

# define callbacks
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping

# save the model weights after each epoch if the validation loss decreased
checkpoint = ModelCheckpoint('best_model.h5', 
                             save_best_only=True, 
                             save_weights_only=True,
                             monitor='val_loss', 
                             mode='min', verbose=1)

# reduce learning rate when the validation loss plateaus
reduce_lr = ReduceLROnPlateau(monitor='val_loss', 
                              factor=0.2, 
                              patience=15, 
                              min_lr=0.0001, verbose=1)

# stop training if the validation loss doesn't improve for 30 consecutive epochs
early_stop = EarlyStopping(monitor='val_loss', patience=20)

# compile the model
model.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy'])

# train the model
history = model.fit(x_train, y_train,
                    validation_data=(x_val, y_val),
                    epochs=50,
                    batch_size=128,
                    callbacks=[checkpoint, reduce_lr, early_stop],
                    class_weight=class_weights
                    )

# evaluate the model on the test set
loss, accuracy = model.evaluate(x_test, y_test, batch_size=1174)

print('Test loss:', loss)
print('Test accuracy:', accuracy)

# evaluate the model on the test set
y_pred = model.predict(x_test)
y_pred_classes = np.round(y_pred)
auc_score = roc_auc_score(y_test, y_pred)
recall = recall_score(y_test, y_pred_classes)
f1 = f1_score(y_test, y_pred_classes)
precision = precision_score(y_test, y_pred_classes)

print('Test AUC:', auc_score)
print('Test Recall:', recall)
print('Test F1-score:', f1)
print('Test Precision:', precision)

# plot confusion matrix
cm = confusion_matrix(y_test, y_pred_classes)
sns.heatmap(cm, annot=True, fmt='g')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.savefig('A5.pdf')
plt.show()

# plot PR curve
precision, recall, _ = precision_recall_curve(y_test, y_pred)
plt.plot(recall, precision)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.savefig('A4.pdf')
plt.show()

# plot ROC curve
fpr, tpr, _ = roc_curve(y_test, y_pred)
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.savefig('A3.pdf')
plt.show()

# plot accuracy and loss curves for training and validation sets
plt.plot(history.history['accuracy'], label='Training Accuracy')
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.savefig('A2.pdf')
plt.show()

plt.plot(history.history['loss'], label='Training Loss')
plt.plot(history.history['val_loss'], label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.savefig('A1.pdf')
plt.show()

model.save('my_model.h5')

from sklearn.metrics import classification_report

# get predicted classes
y_pred = model.predict(x_test)
y_pred_classes = np.round(y_pred)

# create classification report
target_names = ['NORMAL', 'PNEUMONIA']
print(classification_report(y_test, y_pred_classes, target_names=target_names))