Notebooks/Data Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\r\n",
    "import numpy as np\r\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Change working Directory \r\n",
    "os.chdir('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataset\r\n",
    "################### 2020 ########################\r\n",
    "df_2020_train = \"Data//Raw CSV's//2020//train.csv\"\r\n",
    "df_2020_train = pd.read_csv(df_2020_train)\r\n",
    "\r\n",
    "df_2020_test = \"Data//Raw CSV's//2020//test.csv\"\r\n",
    "df_2020_test = pd.read_csv(df_2020_test)\r\n",
    "\r\n",
    "df_2020_duplicates = \"Data//Raw CSV's//2020//Duplicates//2020_Challenge_duplicates.csv\"\r\n",
    "df_2020_duplicates = pd.read_csv(df_2020_duplicates)\r\n",
    "\r\n",
    "################### 2019 ######################\r\n",
    "# GT = Ground Truth\r\n",
    "df_2019_train_GT = \"Data//Raw CSV's//2019//ISIC_2019_Training_GroundTruth.csv\"\r\n",
    "df_2019_train_GT = pd.read_csv(df_2019_train_GT)\r\n",
    "\r\n",
    "# MD = MetaData\r\n",
    "df_2019_train_MD = \"Data//Raw CSV's//2019//Patient MetaData//ISIC_2019_Training_Metadata.csv\"\r\n",
    "df_2019_train_MD = pd.read_csv(df_2019_train_MD)\r\n",
    "\r\n",
    "df_2019_test_MD = \"Data//Raw CSV's//2019//Patient MetaData//ISIC_2019_Test_Metadata.csv\"\r\n",
    "df_2019_test_MD = pd.read_csv(df_2019_test_MD)\r\n",
    "\r\n",
    "\r\n",
    "################### 2018 ######################\r\n",
    "df_2018_valid_GT = \"Data//Raw CSV's//2018//Validation//ISIC2018_Task3_Validation_GroundTruth.csv\"\r\n",
    "df_2018_valid_GT = pd.read_csv(df_2018_valid_GT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform 2020 Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop two columns fromm df_2020_train\n",
    "df_2020_train.drop(['benign_malignant', 'target'], inplace=True, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace Row values\n",
    "df_2020_train = df_2020_train.replace({\n",
    "    'diagnosis': {\n",
    "        'unknown': 'UNK',\n",
    "        'nevus': 'NV',\n",
    "        'melanoma': 'MEL',\n",
    "        'seborrheic keratosis': 'BKL',\n",
    "        'lentigo NOS': 'BKL',\n",
    "        'lichenoid keratosis': 'BKL',\n",
    "        'solar lentigo': 'BKL',\n",
    "        'cafe-au-lait macule': 'UNK',\n",
    "        'atypical melanocytic proliferation': 'UNK'\n",
    "    }\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2020_train = df_2020_train.rename(columns= {\n",
    "    'anatom_site_general_challenge': 'anatom_site_general',\n",
    "    'image_name': 'image',\n",
    "})\n",
    "\n",
    "df_2020_test = df_2020_test.rename(columns= {\n",
    "    'anatom_site_general_challenge': 'anatom_site_general',\n",
    "    'image_name': 'image',\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['UNK', 'NV', 'MEL', 'BKL'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_train.diagnosis.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>male</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>female</td>\n",
       "      <td>50.0</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image  patient_id     sex  age_approx anatom_site_general diagnosis\n",
       "0  ISIC_2637011  IP_7279968    male        45.0           head/neck       UNK\n",
       "1  ISIC_0015719  IP_3075186  female        45.0     upper extremity       UNK\n",
       "2  ISIC_0052212  IP_2842074  female        50.0     lower extremity        NV\n",
       "3  ISIC_0068279  IP_6890425  female        45.0           head/neck       UNK\n",
       "4  ISIC_0074268  IP_8723313  female        55.0     upper extremity       UNK"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2020_train[['anatom_site_general', 'sex']] = df_2020_train[['anatom_site_general', 'sex']].fillna(value = 'unknown')\n",
    "df_2020_train['age_approx'] = df_2020_train['age_approx'].fillna(value = 50)\n",
    "\n",
    "df_2020_test['anatom_site_general'] = df_2020_test['anatom_site_general'].fillna(value = 'unknown')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "image                  0\n",
       "patient_id             0\n",
       "sex                    0\n",
       "age_approx             0\n",
       "anatom_site_general    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_test.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "image                  0\n",
       "patient_id             0\n",
       "sex                    0\n",
       "age_approx             0\n",
       "anatom_site_general    0\n",
       "diagnosis              0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_train.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Removed Duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2020_duplicates_train = df_2020_duplicates[df_2020_duplicates['partition'] == 'train']\n",
    "df_2020_duplicates_test = df_2020_duplicates[df_2020_duplicates['partition'] == 'test']\n",
    "\n",
    "df_2020_duplicates_train.reset_index(inplace=True, drop=True)\n",
    "df_2020_duplicates_test.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33126, 6)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10982, 5)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove_train = list(df_2020_duplicates_train['ISIC_id'].values)\n",
    "remove_test = list(df_2020_duplicates_test['ISIC_id'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2020_train = df_2020_train[~df_2020_train['image'].isin(remove_train)]\n",
    "df_2020_test = df_2020_test[~df_2020_test['image'].isin(remove_test)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10875, 5)\n",
      "(32701, 6)\n"
     ]
    }
   ],
   "source": [
    "print(df_2020_test.shape)\n",
    "print(df_2020_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0052060</td>\n",
       "      <td>IP_3579794</td>\n",
       "      <td>male</td>\n",
       "      <td>70.0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0052349</td>\n",
       "      <td>IP_7782715</td>\n",
       "      <td>male</td>\n",
       "      <td>40.0</td>\n",
       "      <td>lower extremity</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0058510</td>\n",
       "      <td>IP_7960270</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>torso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0073313</td>\n",
       "      <td>IP_6375035</td>\n",
       "      <td>female</td>\n",
       "      <td>50.0</td>\n",
       "      <td>torso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0073502</td>\n",
       "      <td>IP_0589375</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>lower extremity</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image  patient_id     sex  age_approx anatom_site_general\n",
       "0  ISIC_0052060  IP_3579794    male        70.0             unknown\n",
       "1  ISIC_0052349  IP_7782715    male        40.0     lower extremity\n",
       "2  ISIC_0058510  IP_7960270  female        55.0               torso\n",
       "3  ISIC_0073313  IP_6375035  female        50.0               torso\n",
       "4  ISIC_0073502  IP_0589375  female        45.0     lower extremity"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform 2019 Dataset\r\n",
    " \r\n",
    "*Note*: 2019 Dataset is a strick Superset of 2018 dataset. Which means that all the 2018 data is included in the 2019 dataset. [Refer](https://forum.isic-archive.com/t/isic-2018-vs-isic-2019-dataset/1102)\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>MEL</th>\n",
       "      <th>NV</th>\n",
       "      <th>BCC</th>\n",
       "      <th>AK</th>\n",
       "      <th>BKL</th>\n",
       "      <th>DF</th>\n",
       "      <th>VASC</th>\n",
       "      <th>SCC</th>\n",
       "      <th>UNK</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0000001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0000002</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0000003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0000004</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image  MEL   NV  BCC   AK  BKL   DF  VASC  SCC  UNK\n",
       "0  ISIC_0000000  0.0  1.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0\n",
       "1  ISIC_0000001  0.0  1.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0\n",
       "2  ISIC_0000002  1.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0\n",
       "3  ISIC_0000003  0.0  1.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0\n",
       "4  ISIC_0000004  1.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_train_GT.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert Wide to Long Format\n",
    "df_2019_train_GT_transformed = pd.melt(df_2019_train_GT, id_vars='image', var_name='diagnosis', value_name='value').\\\n",
    "    sort_values('image').\\\n",
    "    reset_index(drop=True)\n",
    "\n",
    "# Select rows when diagnosis == 1\n",
    "df_2019_train_GT_transformed = df_2019_train_GT_transformed[df_2019_train_GT_transformed['value'] == 1]\n",
    "df_2019_train_GT_transformed.drop('value', inplace=True, axis = 1)\n",
    "df_2019_train_GT_transformed.reset_index(inplace = True, drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0000000</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0000001</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0000002</td>\n",
       "      <td>MEL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0000003</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0000004</td>\n",
       "      <td>MEL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image diagnosis\n",
       "0  ISIC_0000000        NV\n",
       "1  ISIC_0000001        NV\n",
       "2  ISIC_0000002       MEL\n",
       "3  ISIC_0000003        NV\n",
       "4  ISIC_0000004       MEL"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_train_GT_transformed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine Train metadata and groundtruth variable.\n",
    "df_2019_train_GT_combined = df_2019_train_GT_transformed.merge(df_2019_train_MD, on='image')\n",
    "df_2019_train_GT_combined.drop('lesion_id', inplace=True, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general</th>\n",
       "      <th>sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0000000</td>\n",
       "      <td>NV</td>\n",
       "      <td>55.0</td>\n",
       "      <td>anterior torso</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0000001</td>\n",
       "      <td>NV</td>\n",
       "      <td>30.0</td>\n",
       "      <td>anterior torso</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0000002</td>\n",
       "      <td>MEL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0000003</td>\n",
       "      <td>NV</td>\n",
       "      <td>30.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0000004</td>\n",
       "      <td>MEL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>posterior torso</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image diagnosis  age_approx anatom_site_general     sex\n",
       "0  ISIC_0000000        NV        55.0      anterior torso  female\n",
       "1  ISIC_0000001        NV        30.0      anterior torso  female\n",
       "2  ISIC_0000002       MEL        60.0     upper extremity  female\n",
       "3  ISIC_0000003        NV        30.0     upper extremity    male\n",
       "4  ISIC_0000004       MEL        80.0     posterior torso    male"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_train_GT_combined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2019_train_GT_combined[['anatom_site_general', 'sex']] = df_2019_train_GT_combined[['anatom_site_general', 'sex']].\\\n",
    "    fillna(value = 'unknown')\n",
    "df_2019_train_GT_combined['age_approx'] = df_2019_train_GT_combined['age_approx'].fillna(value = 55)\n",
    "\n",
    "df_2019_test_MD[['anatom_site_general', 'sex']] = df_2019_test_MD[['anatom_site_general', 'sex']].fillna(value = 'unknown')\n",
    "df_2019_test_MD['age_approx'] = df_2019_test_MD['age_approx'].fillna(value = 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "image                  0\n",
       "diagnosis              0\n",
       "age_approx             0\n",
       "anatom_site_general    0\n",
       "sex                    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_train_GT_combined.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "image                  0\n",
       "age_approx             0\n",
       "anatom_site_general    0\n",
       "sex                    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_test_MD.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25331, 5)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_train_GT_combined.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8238, 4)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_test_MD.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform 2018 Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>MEL</th>\n",
       "      <th>NV</th>\n",
       "      <th>BCC</th>\n",
       "      <th>AKIEC</th>\n",
       "      <th>BKL</th>\n",
       "      <th>DF</th>\n",
       "      <th>VASC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0034321</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0034322</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0034323</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0034324</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0034325</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>ISIC_0034519</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>ISIC_0034520</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>ISIC_0034521</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>ISIC_0034522</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>ISIC_0034523</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>193 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            image  MEL   NV  BCC  AKIEC  BKL   DF  VASC\n",
       "0    ISIC_0034321  0.0  1.0  0.0    0.0  0.0  0.0   0.0\n",
       "1    ISIC_0034322  0.0  1.0  0.0    0.0  0.0  0.0   0.0\n",
       "2    ISIC_0034323  0.0  0.0  1.0    0.0  0.0  0.0   0.0\n",
       "3    ISIC_0034324  0.0  1.0  0.0    0.0  0.0  0.0   0.0\n",
       "4    ISIC_0034325  0.0  1.0  0.0    0.0  0.0  0.0   0.0\n",
       "..            ...  ...  ...  ...    ...  ...  ...   ...\n",
       "188  ISIC_0034519  0.0  1.0  0.0    0.0  0.0  0.0   0.0\n",
       "189  ISIC_0034520  0.0  1.0  0.0    0.0  0.0  0.0   0.0\n",
       "190  ISIC_0034521  0.0  0.0  0.0    0.0  1.0  0.0   0.0\n",
       "191  ISIC_0034522  0.0  1.0  0.0    0.0  0.0  0.0   0.0\n",
       "192  ISIC_0034523  0.0  0.0  1.0    0.0  0.0  0.0   0.0\n",
       "\n",
       "[193 rows x 8 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2018_valid_GT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert Wide to Long Format\n",
    "df_2018_valid_GT_transformed = pd.melt(df_2018_valid_GT, id_vars='image', var_name='diagnosis', value_name='value').\\\n",
    "    sort_values('image').\\\n",
    "    reset_index(drop=True)\n",
    "\n",
    "# Select rows when diagnosis == 1\n",
    "df_2018_valid_GT_transformed = df_2018_valid_GT_transformed[df_2018_valid_GT_transformed['value'] == 1]\n",
    "df_2018_valid_GT_transformed.drop('value', inplace=True, axis = 1)\n",
    "df_2018_valid_GT_transformed.reset_index(inplace = True, drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace Row values\n",
    "df_2018_valid_GT_transformed = df_2018_valid_GT_transformed.replace({\n",
    "    'diagnosis': {\n",
    "        'BCC': 'UNK',\n",
    "        'AKIEC': 'UNK',\n",
    "        'DF': 'UNK',\n",
    "        'VASC': 'UNK'\n",
    "    }\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0034321</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0034322</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0034323</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0034324</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0034325</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image diagnosis\n",
       "0  ISIC_0034321        NV\n",
       "1  ISIC_0034322        NV\n",
       "2  ISIC_0034323       UNK\n",
       "3  ISIC_0034324        NV\n",
       "4  ISIC_0034325        NV"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2018_valid_GT_transformed.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combining Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general</th>\n",
       "      <th>sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0000000</td>\n",
       "      <td>NV</td>\n",
       "      <td>55.0</td>\n",
       "      <td>anterior torso</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0000001</td>\n",
       "      <td>NV</td>\n",
       "      <td>30.0</td>\n",
       "      <td>anterior torso</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0000002</td>\n",
       "      <td>MEL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0000003</td>\n",
       "      <td>NV</td>\n",
       "      <td>30.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0000004</td>\n",
       "      <td>MEL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>posterior torso</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image diagnosis  age_approx anatom_site_general     sex\n",
       "0  ISIC_0000000        NV        55.0      anterior torso  female\n",
       "1  ISIC_0000001        NV        30.0      anterior torso  female\n",
       "2  ISIC_0000002       MEL        60.0     upper extremity  female\n",
       "3  ISIC_0000003        NV        30.0     upper extremity    male\n",
       "4  ISIC_0000004       MEL        80.0     posterior torso    male"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_train_GT_combined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>male</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>female</td>\n",
       "      <td>50.0</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image  patient_id     sex  age_approx anatom_site_general diagnosis\n",
       "0  ISIC_2637011  IP_7279968    male        45.0           head/neck       UNK\n",
       "1  ISIC_0015719  IP_3075186  female        45.0     upper extremity       UNK\n",
       "2  ISIC_0052212  IP_2842074  female        50.0     lower extremity        NV\n",
       "3  ISIC_0068279  IP_6890425  female        45.0           head/neck       UNK\n",
       "4  ISIC_0074268  IP_8723313  female        55.0     upper extremity       UNK"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2020_select = df_2020_train[['image', 'sex', 'age_approx', 'anatom_site_general', 'diagnosis']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2019_select = df_2019_train_GT_combined[['image', 'sex', 'age_approx', 'anatom_site_general', 'diagnosis']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PD = Patient Demographic\n",
    "df_2020_2019_PD = df_2020_select.append(df_2019_select, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>male</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>female</td>\n",
       "      <td>50.0</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>NV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image     sex  age_approx anatom_site_general diagnosis\n",
       "0  ISIC_2637011    male        45.0           head/neck       UNK\n",
       "1  ISIC_0015719  female        45.0     upper extremity       UNK\n",
       "2  ISIC_0052212  female        50.0     lower extremity        NV\n",
       "3  ISIC_0068279  female        45.0           head/neck       UNK\n",
       "4  ISIC_0074268  female        55.0     upper extremity       UNK"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_2019_PD.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2020_select_2 = df_2020_train[['image', 'diagnosis']]\n",
    "df_2019_select_2 = df_2019_train_GT_combined[['image', 'diagnosis']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2020_2019_9_label = df_2020_select_2.append(df_2019_select_2, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace Row values\n",
    "df_2020_2019_4_label = df_2020_2019_9_label.replace({\n",
    "    'diagnosis': {\n",
    "        'BCC': 'UNK',\n",
    "        'AK': 'UNK',\n",
    "        'SCC': 'UNK',\n",
    "        'VASC': 'UNK',\n",
    "        'DF': 'UNK'\n",
    "    }\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Note*: we are only selecting 2020 Test set as we do not have cropped images available for the 2019 dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2020_test_PD = df_2020_test[['image', 'sex', 'age_approx', 'anatom_site_general']]\n",
    "df_2020_test_no_PD = df_2020_test[['image']]\n",
    "#df_2019_test_select = df_2019_test_MD[['image', 'sex', 'age_approx', 'anatom_site_general']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PD = Patient Demographic\n",
    "#df_2020_2019_PD_test = df_2020_test_select.append(df_2019_test_select, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0052060</td>\n",
       "      <td>male</td>\n",
       "      <td>70.0</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0052349</td>\n",
       "      <td>male</td>\n",
       "      <td>40.0</td>\n",
       "      <td>lower extremity</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0058510</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>torso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0073313</td>\n",
       "      <td>female</td>\n",
       "      <td>50.0</td>\n",
       "      <td>torso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0073502</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>lower extremity</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image     sex  age_approx anatom_site_general\n",
       "0  ISIC_0052060    male        70.0             unknown\n",
       "1  ISIC_0052349    male        40.0     lower extremity\n",
       "2  ISIC_0058510  female        55.0               torso\n",
       "3  ISIC_0073313  female        50.0               torso\n",
       "4  ISIC_0073502  female        45.0     lower extremity"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_test_PD.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0052060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0052349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0058510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0073313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0073502</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          image\n",
       "0  ISIC_0052060\n",
       "1  ISIC_0052349\n",
       "2  ISIC_0058510\n",
       "3  ISIC_0073313\n",
       "4  ISIC_0073502"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_test_no_PD.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10875, 4)\n",
      "(10875, 1)\n"
     ]
    }
   ],
   "source": [
    "print(df_2020_test_PD.shape)\n",
    "print(df_2020_test_no_PD.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove some of the rows from the CSV as the images are corrupted "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "####### Test ########\r\n",
    "# Test with Patient Demographic\r\n",
    "#df_2020_test_PD.to_csv(save_path + 'test_2020_withPateintDetail.csv', header=True, index=False)\r\n",
    "\r\n",
    "# Test no Patient Demographic\r\n",
    "#df_2020_test_no_PD.to_csv(save_path + 'test_2020_no_PateintDetail.csv', header=True, index=False)\r\n",
    "\r\n",
    "############ Train ##########\r\n",
    "# Train save with pateint Demographic\r\n",
    "df_2020_2019_PD = df_2020_2019_PD[df_2020_2019_PD['image'] != 'ISIC_0066580']\r\n",
    "df_2020_2019_PD.reset_index(inplace=True, drop=True)\r\n",
    "\r\n",
    "# Train save with no pateint Demographic and has 9 labels\r\n",
    "#df_2020_2019_9_label.to_csv(save_path + 'train_2020_and_2019_with_9_Labels.csv', header=True, index=False)\r\n",
    "df_2020_2019_9_label = df_2020_2019_9_label[df_2020_2019_9_label['image'] != 'ISIC_0066580']\r\n",
    "df_2020_2019_9_label.reset_index(inplace=True, drop=True)\r\n",
    "\r\n",
    "# Train save with no pateint Demographic and has 4 labels\r\n",
    "#df_2020_2019_4_label.to_csv(save_path + 'train_2020_and_2019_with_4_Labels.csv', header=True, index=False)\r\n",
    "df_2020_2019_4_label = df_2020_2019_4_label[df_2020_2019_4_label['image'] != 'ISIC_0066580']\r\n",
    "df_2020_2019_4_label.reset_index(inplace=True, drop=True)\r\n",
    "\r\n",
    "# Validation set\r\n",
    "#df_2018_valid_GT_transformed.to_csv(save_path + 'valid_2018_noPateintDetail.csv', header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save to CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(58031, 2)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2020_2019_9_label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_path = \"Data//Processed CSV's//\"\r\n",
    "\r\n",
    "####### Test ########\r\n",
    "# Test save with pateint Demographic\r\n",
    "#df_2020_2019_PD_test.to_csv(save_path + 'test_2020_and_2019_withPateintDetail.csv', header=True, index=False)\r\n",
    "\r\n",
    "# Test with Patient Demographic\r\n",
    "df_2020_test_PD.to_csv(save_path + 'test_2020_withPateintDetail.csv', header=True, index=False)\r\n",
    "\r\n",
    "# Test no Patient Demographic\r\n",
    "df_2020_test_no_PD.to_csv(save_path + 'test_2020_no_PateintDetail.csv', header=True, index=False)\r\n",
    "\r\n",
    "############ Train ##########\r\n",
    "# Train save with pateint Demographic\r\n",
    "df_2020_2019_PD.to_csv(save_path + 'train_2020_and_2019_withPateintDetail_9_labels.csv', header=True, index=False)\r\n",
    "\r\n",
    "# Train save with no pateint Demographic and has 9 labels\r\n",
    "df_2020_2019_9_label.to_csv(save_path + 'train_2020_and_2019_with_9_Labels.csv', header=True, index=False)\r\n",
    "\r\n",
    "# Train save with no pateint Demographic and has 4 labels\r\n",
    "df_2020_2019_4_label.to_csv(save_path + 'train_2020_and_2019_with_4_Labels.csv', header=True, index=False)\r\n",
    "\r\n",
    "# Validation set\r\n",
    "df_2018_valid_GT_transformed.to_csv(save_path + 'valid_2018_noPateintDetail.csv', header=True, index=False)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d74d7db319bafd33f7c656d0a982f55c49dd2aede06f5b3f0df9d1774c187a88"
  },
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}