CYPER_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NJCtWHDnDHPa"
      },
      "outputs": [],
      "source": [
        "!pip install shap\n",
        "!pip install statannot\n",
        "from sklearn import decomposition\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.preprocessing import minmax_scale\n",
        "from sklearn.preprocessing import MaxAbsScaler\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "from sklearn.preprocessing import Normalizer\n",
        "from sklearn.preprocessing import QuantileTransformer\n",
        "from sklearn.preprocessing import PowerTransformer\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.pipeline import Pipeline, make_pipeline\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import tree\n",
        "from sklearn import svm\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, ConfusionMatrixDisplay\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import roc_curve, precision_recall_curve, auc, make_scorer, recall_score, accuracy_score, precision_score, confusion_matrix\n",
        "import argparse\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import shap"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Unir os dados gerados\n",
        "descriptors=pd.read_csv(\"/content/descriptors.tsv\", sep=\"\\t\")\n",
        "descriptors_II=pd.read_csv(\"/content/descriptors_II.txt\", sep=\"\\t\")\n",
        "labels=pd.read_csv(\"/content/labels.txt\", sep=\"\\t\")\n",
        "\n",
        "df=pd.merge(descriptors, descriptors_II, on='header', how='outer')\n",
        "df1=df.iloc[:,1:]\n",
        "X_train, X_test, y_train, y_test = train_test_split(df1,labels, train_size = 0.3, random_state = 42, stratify =labels)\n"
      ],
      "metadata": {
        "id": "PRH04mIoDNB2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def best_scalling(X_train,X_test,y_train,y_test,Algorithm):\n",
        "  results=list()\n",
        "  scaler=[StandardScaler(),  MinMaxScaler(),MaxAbsScaler(),  RobustScaler(), QuantileTransformer(output_distribution=\"uniform\"), QuantileTransformer(output_distribution=\"normal\"), Normalizer()]\n",
        "  for k in scaler:\n",
        "    pipe = make_pipeline( k, Algorithm)\n",
        "    pipe.fit(X_train, y_train)\n",
        "    results.append(pipe.score(X_test,y_test))\n",
        "  df=pd.DataFrame(list(zip(['StandardScaler','MinMaxScaler','MaxAbsScaler','RobustScaler', 'Uniform','Normal', 'Normalize'],results)), columns=['Method','Method score'])\n",
        "  return df\n"
      ],
      "metadata": {
        "id": "Ov_Jfj6eG1gz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Gaussian=best_scalling(X_train,X_test,y_train.values.ravel(), y_test.values.ravel(),GaussianNB())\n",
        "RandomForest=best_scalling(X_train,X_test,y_train.values.ravel(), y_test.values.ravel(),RandomForestClassifier(random_state=42))\n",
        "SVM=best_scalling(X_train,X_test,y_train.values.ravel(), y_test.values.ravel(),svm.SVC(random_state=42))\n",
        "Decision_Tree=best_scalling(X_train,X_test,y_train.values.ravel(), y_test.values.ravel(),tree.DecisionTreeClassifier(random_state=42))\n",
        "scales=pd.DataFrame(list(zip(Gaussian['Method'],Gaussian['Method score'], RandomForest['Method score'], SVM['Method score'], Decision_Tree['Method score'])), columns=['Escalling method','GaussianNB','Random Forest','SVM','Decission tree'])"
      ],
      "metadata": {
        "id": "7KBR-gn4H8Dt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "scales.melted=pd.melt(scales, id_vars=['Escalling method'], value_vars=['GaussianNB', 'Random Forest','SVM','Decission tree'])\n",
        "g=sns.catplot(\n",
        "    data=scales.melted, kind=\"bar\",\n",
        "    x=\"Escalling method\", y=\"value\", hue=\"variable\", palette=\"tab10\", height=6\n",
        ")\n",
        "g.despine(left=True)\n",
        "g.set_axis_labels(\"Scalling method\", \"Accuracy \")\n",
        "g.legend.set_title(\"\")\n",
        "plt.xticks(rotation=45)"
      ],
      "metadata": {
        "id": "rA7RL-XUH-sZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline_SVM = Pipeline(steps=[('scaler',QuantileTransformer(output_distribution=\"uniform\")),('SVM_estimator',svm.SVC(random_state=42))])\n",
        "\n",
        "pipeline_RF=Pipeline(steps=[('scaler',StandardScaler()),('RF_estimator',RandomForestClassifier(random_state=42))])\n",
        "\n",
        "pipeline_DT=Pipeline(steps=[('scaler',StandardScaler()),('DT_estimator',tree.DecisionTreeClassifier(random_state=42))])\n",
        "\n",
        "pipeline_GNB=Pipeline(steps=[('scaler',MinMaxScaler()),('GNB_estimator',GaussianNB())])\n",
        "\n",
        "searches=[pipeline_GNB,pipeline_RF,pipeline_DT,pipeline_SVM]\n",
        "scores=list()\n",
        "algo=['GaussianNB', 'Random Forest', 'Decision Tress', 'SVM']\n",
        "for clf in searches:\n",
        "  scores.append(cross_val_score(clf,X_train,y_train.values.ravel(), cv=5).mean())\n",
        "df_Train = pd.DataFrame({'Algorithm': algo, 'Train data': scores}).set_index('Algorithm')"
      ],
      "metadata": {
        "id": "eywPEMGXIcEv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "searches=[pipeline_GNB,pipeline_RF,pipeline_DT,pipeline_SVM]\n",
        "algo=['GaussianNB', 'Random Forest', 'Decision Tress', 'SVM']\n",
        "scores = list()\n",
        "for clf in searches:\n",
        "    clf = clf.fit(X_train, y_train.values.ravel())\n",
        "    y_pred = clf.predict(X_test)\n",
        "    scores.append(accuracy_score(y_test, y_pred))\n",
        "\n",
        "df_Test  = pd.DataFrame({'Algorithm': algo, 'Accuracy test': scores}).set_index('Algorithm')"
      ],
      "metadata": {
        "id": "furay63AJfBw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "\n",
        "df = pd.merge(df_Train,df_Test,\n",
        "               on='Algorithm')\n",
        "\n",
        "df.plot( kind= 'bar' , secondary_y= 'Accuracy test' , rot= 0, figsize=(10,10))\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "asntX5qLJm2R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "searches=[pipeline_GNB,pipeline_RF,pipeline_DT,pipeline_SVM]\n",
        "scores=list()\n",
        "\n",
        "algo=['GaussianNB', 'Random Forest', 'Decision Tress', 'SVM']\n",
        "scores = list()\n",
        "\n",
        "for clf in searches:\n",
        "    clf = clf.fit(X_train, y_train.values.ravel())\n",
        "    y_pred = clf.predict(X_test)\n",
        "    scores.append(precision_score(y_test, y_pred, average='macro'))\n",
        "\n",
        "df_Test_precision  = pd.DataFrame({'Algorithm': algo, 'Precision test': scores}).set_index('Algorithm')\n",
        "\n",
        "df2 = pd.merge(df_Train,df_Test_precision,\n",
        "               on='Algorithm')\n",
        "\n",
        "df2.plot( kind= 'bar' , secondary_y= 'Precision test' , rot= 0, figsize=(10,10))\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "OhtJGLhutfxp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "searches=[pipeline_GNB,pipeline_RF,pipeline_DT,pipeline_SVM]\n",
        "scores=list()\n",
        "\n",
        "algo=['GaussianNB', 'Random Forest', 'Decision Tress', 'SVM']\n",
        "scores = list()\n",
        "from sklearn.metrics import matthews_corrcoef\n",
        "for clf in searches:\n",
        "    clf = clf.fit(X_train, y_train.values.ravel())\n",
        "    y_pred = clf.predict(X_test)\n",
        "    scores.append(matthews_corrcoef(y_test, y_pred))\n",
        "\n",
        "df_Test_MCC  = pd.DataFrame({'Algorithm': algo, 'MCC test': scores}).set_index('Algorithm')\n",
        "\n",
        "df3 = pd.merge(df_Train,df_Test_MCC,\n",
        "               on='Algorithm')\n",
        "\n",
        "df3.plot( kind= 'bar' , secondary_y= 'MCC test' , rot= 0, figsize=(10,10))\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "liKj1HXyyumQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline_SVM = Pipeline(steps=[('scaler',QuantileTransformer(output_distribution=\"uniform\")),('SVM_estimator',svm.SVC(random_state=42))])\n",
        "\n",
        "pipeline_RF=Pipeline(steps=[('scaler',StandardScaler()),('RF_estimator',RandomForestClassifier(random_state=42))])\n",
        "\n",
        "pipeline_DT=Pipeline(steps=[('scaler',StandardScaler()),('DT_estimator',tree.DecisionTreeClassifier(random_state=42))])\n",
        "\n",
        "pipeline_GNB=Pipeline(steps=[('scaler',MinMaxScaler()),('GNB_estimator',GaussianNB())])\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "\n",
        "lb = LabelBinarizer()\n",
        "lb.fit(y_train)\n",
        "\n",
        "searches=[pipeline_GNB,pipeline_RF,pipeline_DT,pipeline_SVM]\n",
        "scores=list()\n",
        "algo=['GaussianNB', 'Random Forest', 'Decision Tress', 'SVM']\n",
        "for clf in searches:\n",
        "  scores.append(cross_val_score(clf,X_train,y_train.values.ravel(), cv=5, scoring ='roc_auc'))\n",
        "df_rocAUC = pd.DataFrame({'Algorithm': algo, 'Train data': scores}).set_index('Algorithm')\n",
        "df_rocAUC"
      ],
      "metadata": {
        "id": "CQJ4oAeJ7qwv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "searches=[pipeline_GNB,pipeline_RF,pipeline_DT,pipeline_SVM]\n",
        "scores=list()\n",
        "algo=['GaussianNB', 'Random Forest', 'Decision Tress', 'SVM']\n",
        "for clf in searches:\n",
        "  scores.append(cross_val_score(clf,X_train,y_train.values.ravel(), cv=5, scoring ='accuracy'))\n",
        "df_accuracy = pd.DataFrame(scores,columns=['c1','c2','c3','c4','c5'])\n",
        "df_accuracy['Algorithm'] = ['GaussianNB','Random Forest','Decision Tress','SVM']\n",
        "df_accuracy.m = pd.melt(df_accuracy,id_vars=['Algorithm'],var_name='cross validation round',value_name='Accuracy')"
      ],
      "metadata": {
        "id": "j0VRrIdd9JyI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_accuracy.m"
      ],
      "metadata": {
        "id": "PAyVVY1z_S12"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from statannot import add_stat_annotation\n",
        "x = 'Algorithm'\n",
        "y = 'Accuracy'\n",
        "ax = sns.barplot(data=df_accuracy.m, x=x, y=y, capsize=0.25)\n",
        "ax.bar_label(ax.containers[0], fmt='%.1f', label_type='center')\n",
        "add_stat_annotation(ax, data=df_accuracy.m, x=x, y=y,\n",
        "                    box_pairs=[(\"GaussianNB\", \"Random Forest\"), (\"Random Forest\", \"Decision Tress\"), (\"GaussianNB\", \"Decision Tress\"),(\"Random Forest\",\"SVM\"),('GaussianNB','SVM'),('Decision Tress','SVM')],\n",
        "                    test='Mann-Whitney', text_format='star', loc='outside', verbose=2)"
      ],
      "metadata": {
        "id": "hFr0CctfB8Gz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "searches=[pipeline_GNB,pipeline_RF,pipeline_DT,pipeline_SVM]\n",
        "scores=list()\n",
        "algo=['GaussianNB', 'Random Forest', 'Decision Tress', 'SVM']\n",
        "for clf in searches:\n",
        "  scores.append(cross_val_score(clf,X_train,y_train.values.ravel(), cv=5, scoring ='roc_auc'))\n",
        "df_rocAUC = pd.DataFrame(scores,columns=['c1','c2','c3','c4','c5'])\n",
        "df_rocAUC['Algorithm'] = ['GaussianNB','Random Forest','Decision Tress','SVM']\n",
        "df_rocAUC.m = pd.melt(df_rocAUC,id_vars=['Algorithm'],var_name='cross validation round',value_name='ROC-AUC')\n",
        "from statannot import add_stat_annotation\n",
        "x = 'Algorithm'\n",
        "y = 'ROC-AUC'\n",
        "ax = sns.barplot(data=df_rocAUC.m, x=x, y=y, capsize = 0.25)\n",
        "ax.bar_label(ax.containers[0], fmt='%.1f', label_type='center')\n",
        "add_stat_annotation(ax, data=df_rocAUC.m, x=x, y=y,\n",
        "                    box_pairs=[(\"GaussianNB\", \"Random Forest\"), (\"Random Forest\", \"Decision Tress\"), (\"GaussianNB\", \"Decision Tress\"),(\"Random Forest\",\"SVM\"),('GaussianNB','SVM'),('Decision Tress','SVM')],\n",
        "                    test='Mann-Whitney', text_format='star', loc='outside', verbose=2)"
      ],
      "metadata": {
        "id": "9FqE-Hf9Hijm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "searches=[pipeline_GNB,pipeline_RF,pipeline_DT,pipeline_SVM]\n",
        "scores=list()\n",
        "algo=['GaussianNB', 'Random Forest', 'Decision Tress', 'SVM']\n",
        "for clf in searches:\n",
        "  scores.append(cross_val_score(clf,X_train,y_train.values.ravel(), cv=5, scoring ='matthews_corrcoef'))\n",
        "df_MCC = pd.DataFrame(scores,columns=['c1','c2','c3','c4','c5'])\n",
        "df_MCC['Algorithm'] = ['GaussianNB','Random Forest','Decision Tress','SVM']\n",
        "df_MCC.m = pd.melt(df_MCC,id_vars=['Algorithm'],var_name='cross validation round',value_name='MCC')\n",
        "from statannot import add_stat_annotation\n",
        "x = 'Algorithm'\n",
        "y = 'MCC'\n",
        "ax = sns.barplot(data=df_MCC.m, x=x, y=y, capsize = 0.25)\n",
        "ax.bar_label(ax.containers[0], fmt='%.1f', label_type='center')\n",
        "add_stat_annotation(ax, data=df_MCC.m, x=x, y=y,\n",
        "                    box_pairs=[(\"GaussianNB\", \"Random Forest\"), (\"Random Forest\", \"Decision Tress\"), (\"GaussianNB\", \"Decision Tress\"),(\"Random Forest\",\"SVM\"),('GaussianNB','SVM'),('Decision Tress','SVM')],\n",
        "                    test='Mann-Whitney', text_format='star', loc='outside', verbose=2)"
      ],
      "metadata": {
        "id": "POyLCIv1LQwo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Stablishing the parameters to be tested\n",
        "\n",
        "param_grid_RF={'RF_estimator__max_depth':[None,2,4,8,10,12], #None\n",
        "              'RF_estimator__min_samples_split': [2, 5, 10],\n",
        "               'RF_estimator__criterion':['gini','entropy','log_loss'],\n",
        "               'RF_estimator__max_features':['sqrt','log2',None],\n",
        "               'RF_estimator__bootstrap':[False,True]}\n",
        "\n",
        "pipeline_RF=Pipeline(steps=[('scaler',StandardScaler()), ('RF_estimator',RandomForestClassifier(random_state=42))])\n",
        "\n",
        "RF_search=GridSearchCV(pipeline_RF, param_grid_RF, refit=True, cv=5)\n",
        "best_model_RF = RF_search.fit(X_train,y_train.values.ravel())"
      ],
      "metadata": {
        "id": "-5IoeVbZJpz2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(best_model_RF.best_params_, best_model_RF.best_score_)"
      ],
      "metadata": {
        "id": "bxBtCntjLFsh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred=best_model_RF.predict(X_test)\n",
        "print(classification_report(y_test, y_pred))"
      ],
      "metadata": {
        "id": "zj5XzyekUcO0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "clf_report = classification_report(y_test,\n",
        "                                   y_pred,\n",
        "                                   target_names=['WT','NWT'],\n",
        "                                   output_dict=True)\n",
        "sns.heatmap(pd.DataFrame(clf_report).iloc[:-1, :].T, annot=True)"
      ],
      "metadata": {
        "id": "630gzK7I63Gf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ConfusionMatrixDisplay.from_predictions(y_test, y_pred)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "wk2f5QX4q8xe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "RF=Pipeline(steps=[('scaler',StandardScaler()), ('RF_estimator',RandomForestClassifier(random_state=42,\n",
        "                                                                                                bootstrap = False,\n",
        "                                                                                                criterion = 'entropy', max_depth = 2, max_features = 'log2', min_samples_split =2))])"
      ],
      "metadata": {
        "id": "DK63K8oj56rE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_RF = RF.fit(X_train,y_train.values.ravel())"
      ],
      "metadata": {
        "id": "eapDXrle9K4x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "feature_names = model_RF[:-1].get_feature_names_out()\n",
        "\n",
        "mdi_importances = pd.Series(\n",
        "    model_RF[-1].feature_importances_, index=feature_names\n",
        ").sort_values(ascending=False)\n",
        "df_importances = mdi_importances[:20]\n",
        "ax = df_importances.plot.barh()\n",
        "ax.set_title(\"Mean Decrease in Impurity (MDI)\")\n",
        "ax.figure.tight_layout()\n",
        "ax.set(xlabel='Relative importance (%)', ylabel='Protein descriptor')"
      ],
      "metadata": {
        "id": "tXTLzvx29Tfx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cor_matrix = df1.corr(method='spearman').abs()\n",
        "print(cor_matrix)"
      ],
      "metadata": {
        "id": "uNzyPZUAhyeb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "upper = cor_matrix.where(np.triu(np.ones(cor_matrix.shape),k=1).astype(np.bool))\n",
        "to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]\n",
        "df.reduced=df1.drop(to_drop, axis=1)"
      ],
      "metadata": {
        "id": "5Aamv26-iASr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(df.reduced,labels, train_size = 0.3, random_state = 42, stratify =labels)"
      ],
      "metadata": {
        "id": "yOC2kzxzjF39"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "RF=Pipeline(steps=[('scaler',StandardScaler()), ('RF_estimator',RandomForestClassifier(random_state=42,\n",
        "                                                                                                bootstrap = False,\n",
        "                                                                                                criterion = 'entropy', max_depth = 2, max_features = 'log2', min_samples_split =2))])\n",
        "model_RF = RF.fit(X_train,y_train.values.ravel())"
      ],
      "metadata": {
        "id": "S9U8rtnKjM9O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred=model_RF.predict(X_test)\n",
        "print(classification_report(y_test, y_pred))"
      ],
      "metadata": {
        "id": "HsrUX3LUXYc4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.inspection import permutation_importance\n",
        "\n",
        "result_train = permutation_importance(\n",
        "    model_RF, X_train, y_train, n_repeats=5, random_state=42, n_jobs=10\n",
        ")"
      ],
      "metadata": {
        "id": "F-UWt27t9UVo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_importances_idx = result_train.importances_mean.argsort()\n",
        "\n",
        "importances = pd.DataFrame(\n",
        "    result_train.importances[sorted_importances_idx].T,\n",
        "    columns=X_train.columns[sorted_importances_idx],\n",
        ")\n",
        "\n",
        "importances = importances.loc[(importances.sum(axis=1) != 0), (importances.sum(axis=0) != 0)]\n",
        "\n",
        "ax = importances.plot.box(vert=False, whis=10)\n",
        "ax.set_title(\"Permutation Importances (Train set)\")\n",
        "ax.axvline(x=0, color=\"k\", linestyle=\"--\")\n",
        "ax.set_xlabel(\"Decrease in accuracy score\")\n",
        "ax.figure.tight_layout()"
      ],
      "metadata": {
        "id": "SMsHSCDVFAUD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result_test = permutation_importance(\n",
        "    model_RF, X_test, y_test, n_repeats=5, random_state=42, n_jobs=10\n",
        ")"
      ],
      "metadata": {
        "id": "OIT3J2MxEhC_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_importances_idx = result_test.importances_mean.argsort()\n",
        "\n",
        "importances = pd.DataFrame(\n",
        "    result_test.importances[sorted_importances_idx].T,\n",
        "    columns=X_test.columns[sorted_importances_idx],\n",
        ")\n",
        "\n",
        "\n",
        "importances = importances.loc[(importances.sum(axis=1) != 0), (importances.sum(axis=0) != 0)]\n",
        "\n",
        "ax = importances.plot.box(vert=False, whis=10)\n",
        "ax.set_title(\"Permutation Importances (Test set)\")\n",
        "ax.axvline(x=0, color=\"k\", linestyle=\"--\")\n",
        "ax.set_xlabel(\"Decrease in accuracy score\")\n",
        "ax.figure.tight_layout()\n",
        "\n",
        "sns.set(rc={'figure.figsize':(5,8)})"
      ],
      "metadata": {
        "id": "1vX40FxFEuix"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}