pipeline.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nu6o6ZUJEBLr"
      },
      "source": [
        "# Pipeline for clinical and proteomic data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-TQXb4M2Dh9q"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "\n",
        "from scipy.stats import reciprocal, uniform\n",
        "from sklearn.model_selection import cross_val_predict\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
        "from sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve, f1_score\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.model_selection import train_test_split, RandomizedSearchCV\n",
        "from sklearn.metrics import mean_squared_error, accuracy_score, classification_report, confusion_matrix, roc_auc_score, roc_curve\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.feature_selection import SelectKBest, chi2, RFE, SelectFromModel\n",
        "from sklearn.decomposition import PCA\n",
        "%matplotlib inline\n",
        "from matplotlib.pylab import rcParams\n",
        "rcParams['figure.figsize']= 18, 8\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "np.random.seed(123456)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qg3k6WSN06dh"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xchqQqWgD6gR"
      },
      "source": [
        "Import the biological dataset containing peptide data. Data will be normalized and analyzed through the pipeline phases.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SB_cWStAD1ce"
      },
      "outputs": [],
      "source": [
        "# load dataset\n",
        "data = pd.read_excel('/content/drive/...', sheet_name = '...')\n",
        "# delete unnamed columns\n",
        "data = data.loc[:, ~data.columns.str.contains('^Unnamed')]\n",
        "# data preview\n",
        "data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XJJGLVakFLNi"
      },
      "source": [
        "## Data preprocessing\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# print column names\n",
        "data.columns"
      ],
      "metadata": {
        "id": "zjI4xViSNpP3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "etXGBd18FQM1"
      },
      "outputs": [],
      "source": [
        "# delete unwanted columns\n",
        "data = data.drop(['Name','Gleason'], axis=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "okMbHHUUFXCO"
      },
      "source": [
        "Optionally impute missing values\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bj7rKm0xFghp"
      },
      "outputs": [],
      "source": [
        "# replace not a number with zeroes\n",
        "data = data.replace(np.nan,0.0)\n",
        "# replace infinitive values with zeroes\n",
        "data = data.replace(np.inf,0.0)\n",
        "# report missing values\n",
        "data.isna().sum()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xidfbZx4FpYm"
      },
      "source": [
        "Optionally delete row not having the class variable"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l2WPOmcyFsHD"
      },
      "outputs": [],
      "source": [
        "class_column = 'COLUMN COINTAING CLASS'\n",
        "# delete rows without target value\n",
        "data = data.drop(data[data[class_column] == 0.0].index, axis=0)\n",
        "data = data.reset_index()\n",
        "data = data.drop(['index'], axis = 1)\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now uniform back all zeroes to \"not a number\" values before value imputation"
      ],
      "metadata": {
        "id": "6VpY4xSuuARJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nMMzsRRBFwS5"
      },
      "outputs": [],
      "source": [
        "data = data.replace(0.0, np.nan)\n",
        "data.isna().sum()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IOpGDP9xF9_A"
      },
      "source": [
        "## Missing values imputation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f1VfLZlXF_Qc"
      },
      "outputs": [],
      "source": [
        "for column in data:\n",
        "  data[column] = data[column].fillna(data[column].mean())"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# count how many rows are in each target\n",
        "data[class_column].value_counts()"
      ],
      "metadata": {
        "id": "FaT5ptnDXCZ-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sOS-0cnEGUMM"
      },
      "source": [
        "Let's binarize the class variable: 0 for the first one, 1 for the second one"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ta7t82vPGXjo"
      },
      "outputs": [],
      "source": [
        "data[class_column].loc[data[class_column] == 'FIRST_CLASS_LABEL' ] = 0\n",
        "data[class_column].loc[data[class_column] == 'SECOND_CLASS_LABEL' ] = 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pqlWu4M5G1RD"
      },
      "outputs": [],
      "source": [
        "# output variable\n",
        "Y = data[class_column]\n",
        "# training data\n",
        "X_1 = data.drop([class_column], axis=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ib1CElejG5N0"
      },
      "source": [
        "# Feature selection\n",
        "\n",
        "Select a subset of variables significant for the analysis phases, by using several (and optional) Machine Learning tools:\n",
        "\n",
        "* **Pearson correlation**\n",
        "* **Chi-square test**\n",
        "* **RFE**\n",
        "* **Logistic regression**\n",
        "* **Random Forest**\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hlrd6UQVHUzo"
      },
      "outputs": [],
      "source": [
        "# convert output to float type\n",
        "Y = Y.astype(float)\n",
        "# max number of features\n",
        "num_feats = 20"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# normalization\n",
        "X_scaled = MinMaxScaler().fit_transform(X_1)\n",
        "X_scaled = pd.DataFrame(X_scaled, columns = X_1.columns)"
      ],
      "metadata": {
        "id": "vPWlLRsOrQRR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RhoKHfFxHjzl"
      },
      "source": [
        "#  Pearson correlation\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9RVGgKKyHvuy"
      },
      "outputs": [],
      "source": [
        "def cor_selector(X, Y, num_feats):\n",
        "  cor_list = []\n",
        "  feature_name = X.columns.tolist()\n",
        "\n",
        "  for i in X.columns.tolist():\n",
        "    cor = np.corrcoef(X[i], Y)[0, 1]\n",
        "    cor_list.append(cor)\n",
        "    cor_list = [0 if np.isnan(i) else i for i in cor_list]\n",
        "    cor_feature = X.iloc[:, np.argsort(np.abs(cor_list))[-num_feats:]].columns.tolist()\n",
        "    cor_support = [True if i in cor_feature else False for i in feature_name]\n",
        "\n",
        "  return cor_support, cor_feature\n",
        "\n",
        "cor_support, cor_feature = cor_selector(X_scaled, Y, num_feats)\n",
        "print(str(len(cor_feature)), 'selected features')\n",
        "print(cor_feature)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nL3hRdvCIE0e"
      },
      "source": [
        "# Chi square\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0oPutg-oIOPF"
      },
      "outputs": [],
      "source": [
        "chi_selector = SelectKBest(chi2, k = num_feats)\n",
        "chi_selector.fit(X_scaled, Y)\n",
        "chi_support = chi_selector.get_support()\n",
        "chi_feature = X_scaled.loc[:,chi_support].columns.tolist()\n",
        "print(str(len(chi_feature)), 'selected features')\n",
        "print(chi_feature)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t2kaWZepIVrP"
      },
      "source": [
        "# RFE - Recursive Feature Elimination\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Yw1LScplId6B"
      },
      "outputs": [],
      "source": [
        "rfe_selector = RFE(estimator = LogisticRegression(), n_features_to_select = num_feats, step = 5, verbose = 5)\n",
        "rfe_selector.fit(X_scaled, Y)\n",
        "rfe_support = rfe_selector.get_support()\n",
        "rfe_feature = X_scaled.loc[:,rfe_support].columns.tolist()\n",
        "print(str(len(rfe_feature)), 'selected features')\n",
        "print(rfe_feature)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "51hU81iTIhu9"
      },
      "source": [
        "# Logistic Regression\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RcC8349cJPo3"
      },
      "outputs": [],
      "source": [
        "embedded_lr_selector = SelectFromModel(LogisticRegression(penalty = \"l1\", solver = 'liblinear'), max_features = num_feats)\n",
        "embedded_lr_selector.fit(X_scaled, Y)\n",
        "embedded_lr_support = embedded_lr_selector.get_support()\n",
        "embedded_lr_feature = X_scaled.loc[:,embedded_lr_support].columns.tolist()\n",
        "print(str(len(embedded_lr_feature)), 'selected features')\n",
        "print(embedded_lr_feature)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q4Y9_vlEJQjQ"
      },
      "source": [
        "# Random Forest\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "36-D-3XWKLHQ"
      },
      "outputs": [],
      "source": [
        "embedded_rf_selector = SelectFromModel(RandomForestClassifier(30), max_features = num_feats)\n",
        "embedded_rf_selector.fit(X_scaled, Y)\n",
        "embedded_rf_support = embedded_rf_selector.get_support()\n",
        "embedded_rf_feature = X_scaled.loc[:,embedded_rf_support].columns.tolist()\n",
        "print(str(len(embedded_rf_feature)), 'selected features')\n",
        "print(embedded_rf_feature)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dg-yd5ppKTKz"
      },
      "source": [
        "# Feature selection table\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CMVZebXlKXup"
      },
      "outputs": [],
      "source": [
        "feature_name= X_scaled.columns\n",
        "\n",
        "feature_selection_df = pd.DataFrame({'Feature': feature_name,\n",
        "                                     'Pearson': cor_support,\n",
        "                                     'Chi-2': chi_support,\n",
        "                                     'RFE': rfe_support,\n",
        "                                     'Logistic regression': embedded_lr_support,\n",
        "                                     'Random Forest': embedded_rf_support})\n",
        "\n",
        "feature_selection_df['Total'] = np.sum(feature_selection_df, axis = 1)\n",
        "feature_selection_df = feature_selection_df.sort_values(['Total', 'Feature'], ascending = False)\n",
        "feature_selection_df.index = range(1, len(feature_selection_df) + 1)\n",
        "feature_selection_df[:20]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H5EtGU2lKauW"
      },
      "source": [
        "Now select the columns on which at least 'min_score' feature selection tools agreed to be relevant\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T5fz6FBGKq2_"
      },
      "outputs": [],
      "source": [
        "min_score = 4\n",
        "\n",
        "features = []\n",
        "for row in feature_selection_df.itertuples():\n",
        "  if (row[-1] >= min_score):\n",
        "    features.append(row[1])\n",
        "\n",
        "features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VElC7tM0LxJv"
      },
      "source": [
        "# Machine Learning models training\n",
        "\n",
        "We will adopt a 10-fold cross validation approach and build the training and test sets through the StratifiedKFold class"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# definition of the ML models\n",
        "lr = LogisticRegression()\n",
        "dtree = DecisionTreeClassifier()\n",
        "neigh = KNeighborsClassifier(n_neighbors=3)\n",
        "svclassifier = SVC(C=1.2058449429580245, gamma=0.0870602087830485, probability=True)\n",
        "rf_fit = RandomForestClassifier(n_estimators=8, criterion=\"gini\", min_samples_split=2, bootstrap=True,\n",
        "                                 max_features='auto', random_state=42, min_samples_leaf=1)\n",
        "\n",
        "models = [lr, dtree, neigh, svclassifier, rf_fit]\n",
        "\n",
        "# creation of the StratifiedKFold object for the 10-fold cross-validation strategy\n",
        "num_folds = 10\n",
        "stratified_kfold = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=42)\n",
        "\n",
        "# ML models training\n",
        "for model in models:\n",
        "    all_fpr = []\n",
        "    all_tpr = []\n",
        "    all_f1 = []\n",
        "    all_acc = []\n",
        "    all_sens = []\n",
        "    all_spec = []\n",
        "\n",
        "    model_name = model.__class__.__name__\n",
        "    cross_val_results = cross_val_score(model, X_scaled, Y, cv=stratified_kfold, scoring='accuracy')\n",
        "    all_confusion_matrices = []\n",
        "\n",
        "    for train_index, test_index in stratified_kfold.split(X_scaled, Y):\n",
        "        X_train, X_test = X_scaled.iloc[train_index], X_scaled.iloc[test_index]\n",
        "        y_train, y_test = Y.iloc[train_index], Y.iloc[test_index]\n",
        "\n",
        "        model.fit(X_train, y_train)\n",
        "\n",
        "        y_pred = model.predict(X_test)\n",
        "        # calculates the probability for the positive class\n",
        "        y_prob = model.predict_proba(X_test)[:, 1]\n",
        "        # calculate the ROC curve\n",
        "        fpr, tpr, thresholds = roc_curve(y_test, y_prob)\n",
        "\n",
        "        # Interpola la curva ROC per avere la stessa lunghezza\n",
        "        mean_fpr = np.linspace(0, 1, 100)  # Aggiunto questo per inizializzare mean_fpr\n",
        "        interp_tpr = np.interp(mean_fpr, fpr, tpr)\n",
        "        interp_tpr[0] = 0.0\n",
        "        all_fpr.append(mean_fpr)\n",
        "        all_tpr.append(interp_tpr)\n",
        "\n",
        "        # Calcola le altre metriche\n",
        "        predicted = model.predict(X_test)\n",
        "        cm = confusion_matrix(y_test, predicted)\n",
        "        f1 = f1_score(y_test, predicted)\n",
        "        acc = accuracy_score(y_test, predicted)\n",
        "        sens = cm[0, 0]/(cm[0, 0] + cm[0, 1])\n",
        "        spec = cm[1, 1] / (cm[1, 0] + cm[1, 1])\n",
        "\n",
        "        all_f1.append(f1)\n",
        "        all_acc.append(acc)\n",
        "        all_sens.append(sens)\n",
        "        all_spec.append(spec)\n",
        "\n",
        "    # calculate the average for the ROC curve\n",
        "    mean_fpr = np.mean(all_fpr, axis=0)\n",
        "    mean_tpr = np.mean(all_tpr, axis=0)\n",
        "\n",
        "    # calculate the average for the F1\n",
        "    mean_f1 = np.mean(all_f1)\n",
        "    # add the F1 score to the list\n",
        "    f1_scores.append(mean_f1)\n",
        "\n",
        "    # calculate the area under the ROC curve (AUC)\n",
        "    roc_auc = auc(mean_fpr, mean_tpr)\n",
        "\n",
        "    # plot the averaged ROC curve for the model\n",
        "    plt.plot(mean_fpr, mean_tpr, lw=2, label=f'{model.__class__.__name__} (AUC = {roc_auc:.2f})')\n",
        "\n",
        "    # print all of the other metrices\n",
        "    print(f'{model.__class__.__name__}:')\n",
        "    print(f'  AUC: {np.mean(roc_auc):.2f}')\n",
        "    print(f'  F1: {np.mean(all_f1):.2f}')\n",
        "    print(f'  Accuracy: {np.mean(all_acc):.2f}')\n",
        "    print(f'  Sensitivity: {np.mean(all_sens):.2f}')\n",
        "    print(f'  Specificity: {np.mean(all_spec):.2f}')\n",
        "    print()\n",
        "\n",
        "plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
        "plt.xlim([0.0, 1.0])\n",
        "plt.ylim([0.0, 1.05])\n",
        "plt.xlabel('False Positive Rate')\n",
        "plt.ylabel('True Positive Rate')\n",
        "plt.title('Mean ROC Curve - All Models')\n",
        "plt.legend(loc=\"lower right\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "9tbOacPeCUP_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Voting\n",
        "\n",
        "We will use soft voting (i.e. weighting ML models by their accuracies) and hard voting (i.e. majority of according models)"
      ],
      "metadata": {
        "id": "SMwiWLtlBUCM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import VotingClassifier\n",
        "\n",
        "# define validation dataset (please edit the code)\n",
        "validation_data = ..."
      ],
      "metadata": {
        "id": "mfvV6-dKebwj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# soft voting\n",
        "s_voting = VotingClassifier(estimators=[('Rf', rf_fit),\n",
        "                                        ('Lr', lr),\n",
        "                                        ('Knn', neigh),\n",
        "                                        ('Svc', svclassifier),\n",
        "                                        ('Dtc', dtree)],\n",
        "                            voting='soft',\n",
        "                            weights=f1_scores)\n",
        "s_voting = s_voting.fit(data, Y)\n",
        "soft_voting = s_voting.predict(validation_data)"
      ],
      "metadata": {
        "id": "XY4nxW5SegUC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# hard voting\n",
        "h_voting = VotingClassifier( estimators=[ ('Rf', rf_fit),\n",
        "                                          ('Lr', lr),\n",
        "                                          ('Knn', neigh),\n",
        "                                          ('Svc', svclassifier),\n",
        "                                          ('Dtc', dtree)],\n",
        "                             voting='hard')\n",
        "h_voting = h_voting.fit(data, Y)\n",
        "hard_voting = h_voting.predict(validation_data)"
      ],
      "metadata": {
        "id": "F09GyNnCgGd4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Voting tables\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "eXFgSXK_Ccqi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tab_voting = pd.concat([X, soft_voting, hard_voting], axis = 1)\n",
        "tab_voting"
      ],
      "metadata": {
        "id": "QhRY39WngY3Z"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}