Sprint4_BestChallenge_icbeb_CatBoost.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sprint4: BestChallenge_icbeb_CatBoost.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/M-torki/ECG-Classification/blob/main/Sprint4_BestChallenge_icbeb_CatBoost.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZQhcIBu7oGKn"
      },
      "source": [
        "This notebook is based on the paper: \n",
        "\n",
        "**[Deep Learning for ECG Analysis: Benchmarks and Insights from PTB-XL](https://ieeexplore.ieee.org/document/9190034)**\n",
        "\n",
        "The related codes are available on this [Github](https://github.com/helme/ecg_ptbxl_benchmarking/) \n",
        "\n",
        "link to [PTB XL](https://physionet.org/content/ptb-xl/1.0.1/) database, \n",
        "link to [ICBEB2018](http://2018.icbeb.org/Challenge.html) database"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wORp-jjQCMX-",
        "outputId": "ee0e2e71-a74f-49d1-be03-3763e330aa1a"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y_D850nzCiSq",
        "outputId": "c8ee260c-865d-4377-8a4f-a0ae35112e63"
      },
      "source": [
        "cd /gdrive/MyDrive/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/gdrive/MyDrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1144sRmJVHVx",
        "outputId": "c686f66e-a097-4241-ec90-c07906fc8ba3"
      },
      "source": [
        "!pip install wfdb"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wfdb\n",
            "  Downloading wfdb-3.4.0-py3-none-any.whl (137 kB)\n",
            "\u001b[?25l\r\u001b[K     |██▍                             | 10 kB 25.0 MB/s eta 0:00:01\r\u001b[K     |████▊                           | 20 kB 17.9 MB/s eta 0:00:01\r\u001b[K     |███████▏                        | 30 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 40 kB 13.3 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 51 kB 7.3 MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 61 kB 8.5 MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 71 kB 8.7 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 81 kB 8.8 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 92 kB 8.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 102 kB 7.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 112 kB 7.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 122 kB 7.8 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 133 kB 7.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 137 kB 7.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.10.1 in /usr/local/lib/python3.7/dist-packages (from wfdb) (1.19.5)\n",
            "Requirement already satisfied: matplotlib>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wfdb) (3.2.2)\n",
            "Requirement already satisfied: cycler>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from wfdb) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from wfdb) (1.3.1)\n",
            "Requirement already satisfied: urllib3>=1.22 in /usr/local/lib/python3.7/dist-packages (from wfdb) (1.24.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from wfdb) (1.0.1)\n",
            "Requirement already satisfied: chardet>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from wfdb) (3.0.4)\n",
            "Requirement already satisfied: pandas>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from wfdb) (1.1.5)\n",
            "Requirement already satisfied: python-dateutil>=2.4.2 in /usr/local/lib/python3.7/dist-packages (from wfdb) (2.8.1)\n",
            "Requirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.7/dist-packages (from wfdb) (0.22.2.post1)\n",
            "Requirement already satisfied: requests>=2.8.1 in /usr/local/lib/python3.7/dist-packages (from wfdb) (2.23.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.4 in /usr/local/lib/python3.7/dist-packages (from wfdb) (2.4.7)\n",
            "Collecting threadpoolctl>=1.0.0\n",
            "  Downloading threadpoolctl-2.2.0-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from wfdb) (2018.9)\n",
            "Requirement already satisfied: certifi>=2016.8.2 in /usr/local/lib/python3.7/dist-packages (from wfdb) (2021.5.30)\n",
            "Requirement already satisfied: idna>=2.2 in /usr/local/lib/python3.7/dist-packages (from wfdb) (2.10)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from wfdb) (1.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10.0->wfdb) (1.15.0)\n",
            "Installing collected packages: threadpoolctl, wfdb\n",
            "Successfully installed threadpoolctl-2.2.0 wfdb-3.4.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "amteMY4fRNlj"
      },
      "source": [
        "# !git clone https://github.com/helme/ecg_ptbxl_benchmarking/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1ig75yuxRWEE",
        "outputId": "cc256432-560c-4091-ab61-9aece2ea74d8"
      },
      "source": [
        "cd ./ecg_ptbxl_benchmarking/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/gdrive/.shortcut-targets-by-id/1j3ncHY23bba4nXCRAt52Fr8YgGZpml9-/ecg_ptbxl_benchmarking\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4ytFYZfLRnMp",
        "outputId": "fa8e3d3c-2b1b-4777-cfb4-cbfd54542020"
      },
      "source": [
        "cd code/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/gdrive/.shortcut-targets-by-id/1j3ncHY23bba4nXCRAt52Fr8YgGZpml9-/ecg_ptbxl_benchmarking/code\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w3qSI1C11wkq",
        "cellView": "form"
      },
      "source": [
        "#@title utils\n",
        "import os\n",
        "import sys\n",
        "import re\n",
        "import glob\n",
        "import pickle\n",
        "import copy\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm\n",
        "import wfdb\n",
        "import ast\n",
        "from sklearn.metrics import classification_report, fbeta_score, roc_auc_score, roc_curve, roc_curve, auc\n",
        "from sklearn.preprocessing import StandardScaler, MultiLabelBinarizer\n",
        "from matplotlib.axes._axes import _log as matplotlib_axes_logger\n",
        "import warnings\n",
        "\n",
        "# EVALUATION STUFF\n",
        "def generate_results(idxs, y_true, y_pred, thresholds):\n",
        "    return evaluate_experiment(y_true[idxs], y_pred[idxs], thresholds)\n",
        "\n",
        "def evaluate_experiment(y_true, y_pred, thresholds=None):\n",
        "    results = {}\n",
        "\n",
        "    if not thresholds is None:\n",
        "        # binary predictions\n",
        "        y_pred_binary = apply_thresholds(y_pred, thresholds)\n",
        "        # PhysioNet/CinC Challenges metrics\n",
        "        challenge_scores = challenge_metrics(y_true, y_pred_binary, beta1=2, beta2=2)\n",
        "        results['F_beta_macro'] = challenge_scores['F_beta_macro']\n",
        "        results['G_beta_macro'] = challenge_scores['G_beta_macro']\n",
        "\n",
        "    # label based metric\n",
        "    results['macro_auc'] = roc_auc_score(y_true, y_pred, average='macro')\n",
        "    \n",
        "    df_result = pd.DataFrame(results, index=[0])\n",
        "    return df_result\n",
        "\n",
        "def challenge_metrics(y_true, y_pred, beta1=2, beta2=2, class_weights=None, single=False):\n",
        "    f_beta = 0\n",
        "    g_beta = 0\n",
        "    if single: # if evaluating single class in case of threshold-optimization\n",
        "        sample_weights = np.ones(y_true.sum(axis=1).shape)\n",
        "    else:\n",
        "        sample_weights = y_true.sum(axis=1)\n",
        "    for classi in range(y_true.shape[1]):\n",
        "        y_truei, y_predi = y_true[:,classi], y_pred[:,classi]\n",
        "        TP, FP, TN, FN = 0.,0.,0.,0.\n",
        "        for i in range(len(y_predi)):\n",
        "            sample_weight = sample_weights[i]\n",
        "            if y_truei[i]==y_predi[i]==1: \n",
        "                TP += 1./sample_weight\n",
        "            if ((y_predi[i]==1) and (y_truei[i]!=y_predi[i])): \n",
        "                FP += 1./sample_weight\n",
        "            if y_truei[i]==y_predi[i]==0: \n",
        "                TN += 1./sample_weight\n",
        "            if ((y_predi[i]==0) and (y_truei[i]!=y_predi[i])): \n",
        "                FN += 1./sample_weight \n",
        "        f_beta_i = ((1+beta1**2)*TP)/((1+beta1**2)*TP + FP + (beta1**2)*FN)\n",
        "        g_beta_i = (TP)/(TP+FP+beta2*FN)\n",
        "\n",
        "        f_beta += f_beta_i\n",
        "        g_beta += g_beta_i\n",
        "\n",
        "    return {'F_beta_macro':f_beta/y_true.shape[1], 'G_beta_macro':g_beta/y_true.shape[1]}\n",
        "\n",
        "def get_appropriate_bootstrap_samples(y_true, n_bootstraping_samples):\n",
        "    samples=[]\n",
        "    while True:\n",
        "        ridxs = np.random.randint(0, len(y_true), len(y_true))\n",
        "        if y_true[ridxs].sum(axis=0).min() != 0:\n",
        "            samples.append(ridxs)\n",
        "            if len(samples) == n_bootstraping_samples:\n",
        "                break\n",
        "    return samples\n",
        "\n",
        "def find_optimal_cutoff_threshold(target, predicted):\n",
        "    \"\"\" \n",
        "    Find the optimal probability cutoff point for a classification model related to event rate\n",
        "    \"\"\"\n",
        "    fpr, tpr, threshold = roc_curve(target, predicted)\n",
        "    optimal_idx = np.argmax(tpr - fpr)\n",
        "    optimal_threshold = threshold[optimal_idx]\n",
        "    return optimal_threshold\n",
        "\n",
        "def find_optimal_cutoff_thresholds(y_true, y_pred):\n",
        "\treturn [find_optimal_cutoff_threshold(y_true[:,i], y_pred[:,i]) for i in range(y_true.shape[1])]\n",
        "\n",
        "def find_optimal_cutoff_threshold_for_Gbeta(target, predicted, n_thresholds=100):\n",
        "    thresholds = np.linspace(0.00,1,n_thresholds)\n",
        "    scores = [challenge_metrics(target, predicted>t, single=True)['G_beta_macro'] for t in thresholds]\n",
        "    optimal_idx = np.argmax(scores)\n",
        "    return thresholds[optimal_idx]\n",
        "\n",
        "def find_optimal_cutoff_thresholds_for_Gbeta(y_true, y_pred):\n",
        "    print(\"optimize thresholds with respect to G_beta\")\n",
        "    return [find_optimal_cutoff_threshold_for_Gbeta(y_true[:,k][:,np.newaxis], y_pred[:,k][:,np.newaxis]) for k in tqdm(range(y_true.shape[1]))]\n",
        "\n",
        "def apply_thresholds(preds, thresholds):\n",
        "\t\"\"\"\n",
        "\t\tapply class-wise thresholds to prediction score in order to get binary format.\n",
        "\t\tBUT: if no score is above threshold, pick maximum. This is needed due to metric issues.\n",
        "\t\"\"\"\n",
        "\ttmp = []\n",
        "\tfor p in preds:\n",
        "\t\ttmp_p = (p > thresholds).astype(int)\n",
        "\t\tif np.sum(tmp_p) == 0:\n",
        "\t\t\ttmp_p[np.argmax(p)] = 1\n",
        "\t\ttmp.append(tmp_p)\n",
        "\ttmp = np.array(tmp)\n",
        "\treturn tmp\n",
        "\n",
        "# DATA PROCESSING STUFF\n",
        "\n",
        "def load_dataset(path, sampling_rate, release=False):\n",
        "    if path.split('/')[-2] == 'ptbxl':\n",
        "        # load and convert annotation data\n",
        "        Y = pd.read_csv(path+'ptbxl_database.csv', index_col='ecg_id')\n",
        "        Y.scp_codes = Y.scp_codes.apply(lambda x: ast.literal_eval(x))\n",
        "\n",
        "        # Load raw signal data\n",
        "        X = load_raw_data_ptbxl(Y, sampling_rate, path)\n",
        "\n",
        "    elif path.split('/')[-2] == 'ICBEB':\n",
        "        # load and convert annotation data\n",
        "        Y = pd.read_csv(path+'icbeb_database.csv', index_col='ecg_id')\n",
        "        Y.scp_codes = Y.scp_codes.apply(lambda x: ast.literal_eval(x))\n",
        "\n",
        "        # Load raw signal data\n",
        "        X = load_raw_data_icbeb(Y, sampling_rate, path)\n",
        "\n",
        "    return X, Y\n",
        "\n",
        "\n",
        "def load_raw_data_icbeb(df, sampling_rate, path):\n",
        "\n",
        "    if sampling_rate == 100:\n",
        "        if os.path.exists(path + 'raw100.npy'):\n",
        "            data = np.load(path+'raw100.npy', allow_pickle=True)\n",
        "        else:\n",
        "            data = [wfdb.rdsamp(path + 'records100/'+str(f)) for f in tqdm(df.index)]\n",
        "            data = np.array([signal for signal, meta in data])\n",
        "            pickle.dump(data, open(path+'raw100.npy', 'wb'), protocol=4)\n",
        "    elif sampling_rate == 500:\n",
        "        if os.path.exists(path + 'raw500.npy'):\n",
        "            data = np.load(path+'raw500.npy', allow_pickle=True)\n",
        "        else:\n",
        "            data = [wfdb.rdsamp(path + 'records500/'+str(f)) for f in tqdm(df.index)]\n",
        "            data = np.array([signal for signal, meta in data])\n",
        "            pickle.dump(data, open(path+'raw500.npy', 'wb'), protocol=4)\n",
        "    return data\n",
        "\n",
        "def load_raw_data_ptbxl(df, sampling_rate, path):\n",
        "    if sampling_rate == 100:\n",
        "        if os.path.exists(path + 'raw100.npy'):\n",
        "            data = np.load(path+'raw100.npy', allow_pickle=True)\n",
        "        else:\n",
        "            data = [wfdb.rdsamp(path+f) for f in tqdm(df.filename_lr)]\n",
        "            data = np.array([signal for signal, meta in data])\n",
        "            pickle.dump(data, open(path+'raw100.npy', 'wb'), protocol=4)\n",
        "    elif sampling_rate == 500:\n",
        "        if os.path.exists(path + 'raw500.npy'):\n",
        "            data = np.load(path+'raw500.npy', allow_pickle=True)\n",
        "        else:\n",
        "            data = [wfdb.rdsamp(path+f) for f in tqdm(df.filename_hr)]\n",
        "            data = np.array([signal for signal, meta in data])\n",
        "            pickle.dump(data, open(path+'raw500.npy', 'wb'), protocol=4)\n",
        "    return data\n",
        "\n",
        "def compute_label_aggregations(df, folder, ctype):\n",
        "\n",
        "    df['scp_codes_len'] = df.scp_codes.apply(lambda x: len(x))\n",
        "\n",
        "    aggregation_df = pd.read_csv(folder+'scp_statements.csv', index_col=0)\n",
        "\n",
        "    if ctype in ['diagnostic', 'subdiagnostic', 'superdiagnostic']:\n",
        "\n",
        "        def aggregate_all_diagnostic(y_dic):\n",
        "            tmp = []\n",
        "            for key in y_dic.keys():\n",
        "                if key in diag_agg_df.index:\n",
        "                    tmp.append(key)\n",
        "            return list(set(tmp))\n",
        "\n",
        "        def aggregate_subdiagnostic(y_dic):\n",
        "            tmp = []\n",
        "            for key in y_dic.keys():\n",
        "                if key in diag_agg_df.index:\n",
        "                    c = diag_agg_df.loc[key].diagnostic_subclass\n",
        "                    if str(c) != 'nan':\n",
        "                        tmp.append(c)\n",
        "            return list(set(tmp))\n",
        "\n",
        "        def aggregate_diagnostic(y_dic):\n",
        "            tmp = []\n",
        "            for key in y_dic.keys():\n",
        "                if key in diag_agg_df.index:\n",
        "                    c = diag_agg_df.loc[key].diagnostic_class\n",
        "                    if str(c) != 'nan':\n",
        "                        tmp.append(c)\n",
        "            return list(set(tmp))\n",
        "\n",
        "        diag_agg_df = aggregation_df[aggregation_df.diagnostic == 1.0]\n",
        "        if ctype == 'diagnostic':\n",
        "            df['diagnostic'] = df.scp_codes.apply(aggregate_all_diagnostic)\n",
        "            df['diagnostic_len'] = df.diagnostic.apply(lambda x: len(x))\n",
        "        elif ctype == 'subdiagnostic':\n",
        "            df['subdiagnostic'] = df.scp_codes.apply(aggregate_subdiagnostic)\n",
        "            df['subdiagnostic_len'] = df.subdiagnostic.apply(lambda x: len(x))\n",
        "        elif ctype == 'superdiagnostic':\n",
        "            df['superdiagnostic'] = df.scp_codes.apply(aggregate_diagnostic)\n",
        "            df['superdiagnostic_len'] = df.superdiagnostic.apply(lambda x: len(x))\n",
        "    elif ctype == 'form':\n",
        "        form_agg_df = aggregation_df[aggregation_df.form == 1.0]\n",
        "\n",
        "        def aggregate_form(y_dic):\n",
        "            tmp = []\n",
        "            for key in y_dic.keys():\n",
        "                if key in form_agg_df.index:\n",
        "                    c = key\n",
        "                    if str(c) != 'nan':\n",
        "                        tmp.append(c)\n",
        "            return list(set(tmp))\n",
        "\n",
        "        df['form'] = df.scp_codes.apply(aggregate_form)\n",
        "        df['form_len'] = df.form.apply(lambda x: len(x))\n",
        "    elif ctype == 'rhythm':\n",
        "        rhythm_agg_df = aggregation_df[aggregation_df.rhythm == 1.0]\n",
        "\n",
        "        def aggregate_rhythm(y_dic):\n",
        "            tmp = []\n",
        "            for key in y_dic.keys():\n",
        "                if key in rhythm_agg_df.index:\n",
        "                    c = key\n",
        "                    if str(c) != 'nan':\n",
        "                        tmp.append(c)\n",
        "            return list(set(tmp))\n",
        "\n",
        "        df['rhythm'] = df.scp_codes.apply(aggregate_rhythm)\n",
        "        df['rhythm_len'] = df.rhythm.apply(lambda x: len(x))\n",
        "    elif ctype == 'all':\n",
        "        df['all_scp'] = df.scp_codes.apply(lambda x: list(set(x.keys())))\n",
        "\n",
        "    return df\n",
        "\n",
        "def select_data(XX,YY, ctype, min_samples, outputfolder):\n",
        "    # convert multilabel to multi-hot\n",
        "    mlb = MultiLabelBinarizer()\n",
        "\n",
        "    if ctype == 'diagnostic':\n",
        "        X = XX[YY.diagnostic_len > 0]\n",
        "        Y = YY[YY.diagnostic_len > 0]\n",
        "        mlb.fit(Y.diagnostic.values)\n",
        "        y = mlb.transform(Y.diagnostic.values)\n",
        "    elif ctype == 'subdiagnostic':\n",
        "        counts = pd.Series(np.concatenate(YY.subdiagnostic.values)).value_counts()\n",
        "        counts = counts[counts > min_samples]\n",
        "        YY.subdiagnostic = YY.subdiagnostic.apply(lambda x: list(set(x).intersection(set(counts.index.values))))\n",
        "        YY['subdiagnostic_len'] = YY.subdiagnostic.apply(lambda x: len(x))\n",
        "        X = XX[YY.subdiagnostic_len > 0]\n",
        "        Y = YY[YY.subdiagnostic_len > 0]\n",
        "        mlb.fit(Y.subdiagnostic.values)\n",
        "        y = mlb.transform(Y.subdiagnostic.values)\n",
        "    elif ctype == 'superdiagnostic':\n",
        "        counts = pd.Series(np.concatenate(YY.superdiagnostic.values)).value_counts()\n",
        "        counts = counts[counts > min_samples]\n",
        "        YY.superdiagnostic = YY.superdiagnostic.apply(lambda x: list(set(x).intersection(set(counts.index.values))))\n",
        "        YY['superdiagnostic_len'] = YY.superdiagnostic.apply(lambda x: len(x))\n",
        "        X = XX[YY.superdiagnostic_len > 0]\n",
        "        Y = YY[YY.superdiagnostic_len > 0]\n",
        "        mlb.fit(Y.superdiagnostic.values)\n",
        "        y = mlb.transform(Y.superdiagnostic.values)\n",
        "    elif ctype == 'form':\n",
        "        # filter\n",
        "        counts = pd.Series(np.concatenate(YY.form.values)).value_counts()\n",
        "        counts = counts[counts > min_samples]\n",
        "        YY.form = YY.form.apply(lambda x: list(set(x).intersection(set(counts.index.values))))\n",
        "        YY['form_len'] = YY.form.apply(lambda x: len(x))\n",
        "        # select\n",
        "        X = XX[YY.form_len > 0]\n",
        "        Y = YY[YY.form_len > 0]\n",
        "        mlb.fit(Y.form.values)\n",
        "        y = mlb.transform(Y.form.values)\n",
        "    elif ctype == 'rhythm':\n",
        "        # filter \n",
        "        counts = pd.Series(np.concatenate(YY.rhythm.values)).value_counts()\n",
        "        counts = counts[counts > min_samples]\n",
        "        YY.rhythm = YY.rhythm.apply(lambda x: list(set(x).intersection(set(counts.index.values))))\n",
        "        YY['rhythm_len'] = YY.rhythm.apply(lambda x: len(x))\n",
        "        # select\n",
        "        X = XX[YY.rhythm_len > 0]\n",
        "        Y = YY[YY.rhythm_len > 0]\n",
        "        mlb.fit(Y.rhythm.values)\n",
        "        y = mlb.transform(Y.rhythm.values)\n",
        "    elif ctype == 'all':\n",
        "        # filter \n",
        "        counts = pd.Series(np.concatenate(YY.all_scp.values)).value_counts()\n",
        "        counts = counts[counts > min_samples]\n",
        "        YY.all_scp = YY.all_scp.apply(lambda x: list(set(x).intersection(set(counts.index.values))))\n",
        "        YY['all_scp_len'] = YY.all_scp.apply(lambda x: len(x))\n",
        "        # select\n",
        "        X = XX[YY.all_scp_len > 0]\n",
        "        Y = YY[YY.all_scp_len > 0]\n",
        "        mlb.fit(Y.all_scp.values)\n",
        "        y = mlb.transform(Y.all_scp.values)\n",
        "    else:\n",
        "        pass\n",
        "\n",
        "    # save LabelBinarizer\n",
        "    with open(outputfolder+'mlb.pkl', 'wb') as tokenizer:\n",
        "        pickle.dump(mlb, tokenizer)\n",
        "\n",
        "    return X, Y, y, mlb\n",
        "\n",
        "def preprocess_signals(X_train, X_validation, X_test, outputfolder):\n",
        "    # Standardize data such that mean 0 and variance 1\n",
        "    ss = StandardScaler()\n",
        "    ss.fit(np.vstack(X_train).flatten()[:,np.newaxis].astype(float))\n",
        "    \n",
        "    # Save Standardizer data\n",
        "    with open(outputfolder+'standard_scaler.pkl', 'wb') as ss_file:\n",
        "        pickle.dump(ss, ss_file)\n",
        "\n",
        "    return apply_standardizer(X_train, ss), apply_standardizer(X_validation, ss), apply_standardizer(X_test, ss)\n",
        "\n",
        "def apply_standardizer(X, ss):\n",
        "    X_tmp = []\n",
        "    for x in X:\n",
        "        x_shape = x.shape\n",
        "        X_tmp.append(ss.transform(x.flatten()[:,np.newaxis]).reshape(x_shape))\n",
        "    X_tmp = np.array(X_tmp)\n",
        "    return X_tmp\n",
        "\n",
        "\n",
        "# DOCUMENTATION STUFF\n",
        "\n",
        "def generate_ptbxl_summary_table(selection=None, folder='../output/'):\n",
        "\n",
        "    exps = ['exp0', 'exp1', 'exp1.1', 'exp1.1.1', 'exp2', 'exp3']\n",
        "    metric1 = 'macro_auc' \n",
        "\n",
        "    # get models\n",
        "    models = {}\n",
        "    for i, exp in enumerate(exps):\n",
        "        if selection is None:\n",
        "            exp_models = [m.split('/')[-1] for m in glob.glob(folder+str(exp)+'/models/*')]\n",
        "        else:\n",
        "            exp_models = selection\n",
        "        if i == 0:\n",
        "            models = set(exp_models)\n",
        "        else:\n",
        "            models = models.union(set(exp_models))\n",
        "\n",
        "    results_dic = {'Method':[], \n",
        "                'exp0_AUC':[], \n",
        "                'exp1_AUC':[], \n",
        "                'exp1.1_AUC':[], \n",
        "                'exp1.1.1_AUC':[], \n",
        "                'exp2_AUC':[],\n",
        "                'exp3_AUC':[]\n",
        "                }\n",
        "\n",
        "    for m in models:\n",
        "        results_dic['Method'].append(m)\n",
        "        \n",
        "        for e in exps:\n",
        "            \n",
        "            try:\n",
        "                me_res = pd.read_csv(folder+str(e)+'/models/'+str(m)+'/results/te_results.csv', index_col=0)\n",
        "    \n",
        "                mean1 = me_res.loc['point'][metric1]\n",
        "                unc1 = max(me_res.loc['upper'][metric1]-me_res.loc['point'][metric1], me_res.loc['point'][metric1]-me_res.loc['lower'][metric1])\n",
        "\n",
        "                results_dic[e+'_AUC'].append(\"%.3f(%.2d)\" %(np.round(mean1,3), int(unc1*1000)))\n",
        "\n",
        "            except FileNotFoundError:\n",
        "                results_dic[e+'_AUC'].append(\"--\")\n",
        "            \n",
        "            \n",
        "    df = pd.DataFrame(results_dic)\n",
        "    df_index = df[df.Method.isin(['naive', 'ensemble'])]\n",
        "    df_rest = df[~df.Method.isin(['naive', 'ensemble'])]\n",
        "    df = pd.concat([df_rest, df_index])\n",
        "    df.to_csv(folder+'results_ptbxl.csv')\n",
        "\n",
        "    titles = [\n",
        "        '### 1. PTB-XL: all statements',\n",
        "        '### 2. PTB-XL: diagnostic statements',\n",
        "        '### 3. PTB-XL: Diagnostic subclasses',\n",
        "        '### 4. PTB-XL: Diagnostic superclasses',\n",
        "        '### 5. PTB-XL: Form statements',\n",
        "        '### 6. PTB-XL: Rhythm statements'        \n",
        "    ]\n",
        "\n",
        "    # helper output function for markdown tables\n",
        "    our_work = 'https://arxiv.org/abs/2004.13701'\n",
        "    our_repo = 'https://github.com/helme/ecg_ptbxl_benchmarking/'\n",
        "    md_source = ''\n",
        "    for i, e in enumerate(exps):\n",
        "        md_source += '\\n '+titles[i]+' \\n \\n'\n",
        "        md_source += '| Model | AUC &darr; | paper/source | code | \\n'\n",
        "        md_source += '|---:|:---|:---|:---| \\n'\n",
        "        for row in df_rest[['Method', e+'_AUC']].sort_values(e+'_AUC', ascending=False).values:\n",
        "            md_source += '| ' + row[0].replace('fastai_', '') + ' | ' + row[1] + ' | [our work]('+our_work+') | [this repo]('+our_repo+')| \\n'\n",
        "    print(md_source)\n",
        "\n",
        "def ICBEBE_table(selection=None, folder='../output/'):\n",
        "    cols = ['macro_auc', 'F_beta_macro', 'G_beta_macro']\n",
        "\n",
        "    if selection is None:\n",
        "        models = [m.split('/')[-1].split('_pretrained')[0] for m in glob.glob(folder+'exp_ICBEB/models/*')]\n",
        "    else:\n",
        "        models = [] \n",
        "        for s in selection:\n",
        "            #if s != 'Wavelet+NN':\n",
        "                models.append(s)\n",
        "\n",
        "    data = []\n",
        "    for model in models:\n",
        "        me_res = pd.read_csv(folder+'exp_ICBEB/models/'+model+'/results/te_results.csv', index_col=0)\n",
        "        mcol=[]\n",
        "        for col in cols:\n",
        "            mean = me_res.ix['point'][col]\n",
        "            unc = max(me_res.ix['upper'][col]-me_res.ix['point'][col], me_res.ix['point'][col]-me_res.ix['lower'][col])\n",
        "            mcol.append(\"%.3f(%.2d)\" %(np.round(mean,3), int(unc*1000)))\n",
        "        data.append(mcol)\n",
        "    data = np.array(data)\n",
        "\n",
        "    df = pd.DataFrame(data, columns=cols, index=models)\n",
        "    df.to_csv(folder+'results_icbeb.csv')\n",
        "\n",
        "    df_rest = df[~df.index.isin(['naive', 'ensemble'])]\n",
        "    df_rest = df_rest.sort_values('macro_auc', ascending=False)\n",
        "    our_work = 'https://arxiv.org/abs/2004.13701'\n",
        "    our_repo = 'https://github.com/helme/ecg_ptbxl_benchmarking/'\n",
        "\n",
        "    md_source = '| Model | AUC &darr; |  F_beta=2 | G_beta=2 | paper/source | code | \\n'\n",
        "    md_source += '|---:|:---|:---|:---|:---|:---| \\n'\n",
        "    for i, row in enumerate(df_rest[cols].values):\n",
        "        md_source += '| ' + df_rest.index[i].replace('fastai_', '') + ' | ' + row[0] + ' | ' + row[1] + ' | ' + row[2] + ' | [our work]('+our_work+') | [this repo]('+our_repo+')| \\n'\n",
        "    print(md_source)\n",
        "    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zfvtfMA2K2sw"
      },
      "source": [
        "import glob\n",
        "import random\n",
        "import os\n",
        "import argparse\n",
        "import scipy.io as sio\n",
        "from keras import backend as K\n",
        "from sklearn.model_selection import train_test_split\n",
        "import csv\n",
        "import numpy\n",
        "import numpy as np\n",
        "\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import scipy\n",
        "from tensorflow.python.client import device_lib\n",
        "import keras\n",
        "from keras.models import Sequential, load_model\n",
        "from keras.layers import LSTM, GRU, TimeDistributed, Bidirectional, LeakyReLU\n",
        "from keras.layers import Dense, Dropout, Activation, Flatten,  Input, Reshape, GRU, CuDNNGRU\n",
        "from keras.layers import Convolution1D, MaxPool1D, GlobalAveragePooling1D,concatenate,AveragePooling1D\n",
        "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping\n",
        "from keras.models import Model\n",
        "from keras import initializers, regularizers, constraints\n",
        "from keras.layers import Layer\n",
        "import numpy as np\n",
        "from keras.layers.normalization import BatchNormalization\n",
        "from keras import regularizers\n",
        "import scipy.io as sio\n",
        "from os import listdir\n",
        "from keras.optimizers import Adam"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PDXOJhDHqCOz"
      },
      "source": [
        "#sampling frequency=100\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LO4FB4Qbxrfj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d9980a68-2db9-40cb-de63-abc114005100"
      },
      "source": [
        "from utils import utils\n",
        "\n",
        "sampling_frequency=100\n",
        "datafolder= '/gdrive/My Drive/ICBEB/'\n",
        "task='all'\n",
        "outputfolder='../output0/'\n",
        "\n",
        "# Load data\n",
        "data, raw_labels = utils.load_dataset(datafolder, sampling_frequency)\n",
        "# Preprocess label data\n",
        "labels = utils.compute_label_aggregations(raw_labels, datafolder, task)\n",
        "# Select relevant data and convert to one-hot\n",
        "data, labels, Y, _ = utils.select_data(data, labels, task, min_samples=0, outputfolder=outputfolder)\n",
        "\n",
        "# 1-9 for training \n",
        "X_train = data[labels.strat_fold < 10]\n",
        "y_train = Y[labels.strat_fold < 10]\n",
        "# 10 for validation\n",
        "X_val = data[labels.strat_fold == 10]\n",
        "y_val = Y[labels.strat_fold == 10]\n",
        "\n",
        "num_classes = 9         # <=== number of classes in the finetuning dataset\n",
        "input_shape = [1000,12] # <=== shape of samples, [None, 12] in case of different lengths\n",
        "\n",
        "X_train.shape, y_train.shape, X_val.shape, y_val.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((6187,), (6187, 9), (690,), (690, 9))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dxfT8OKac8vC",
        "outputId": "ab170400-089e-495b-e7c0-831bb9d8f52e"
      },
      "source": [
        "X_train[0].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1500, 12)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QtV-zMEK1GOb",
        "outputId": "fad8e983-02e1-4265-f6ca-3d56ab47ec25"
      },
      "source": [
        "X_tr = []\n",
        "for i in range(len(X_train)):\n",
        "    x = []\n",
        "    for j in range(12):\n",
        "        p = X_train[i][:1000,j]\n",
        "        if p.shape[0]!= 1000:\n",
        "            d = abs(p.shape[0]-1000)//2\n",
        "            p = np.pad(p,(d,d))\n",
        "        x.append(p)\n",
        "    X_tr.append(np.transpose(x))\n",
        "X_tr = np.array(X_tr)\n",
        "X_tr.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(6187, 1000, 12)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7qpz5wPKJOxS",
        "outputId": "af7b0075-c271-4601-b5ea-15f873507044"
      },
      "source": [
        "X_te = []\n",
        "for i in range(len(X_val)):\n",
        "    x = []\n",
        "    for j in range(12):\n",
        "        p = X_val[i][:1000,j]\n",
        "        if p.shape[0]!= 1000:\n",
        "            d = abs(p.shape[0]-1000)//2\n",
        "            p = np.pad(p,(d,d))\n",
        "        x.append(p)\n",
        "    X_te.append(np.transpose(x))\n",
        "X_te = np.array(X_te)\n",
        "X_te.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(690, 1000, 12)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t_C8dQjNcCow"
      },
      "source": [
        "def dot_product(x, kernel):\n",
        "    if K.backend() == 'tensorflow':\n",
        "        return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)\n",
        "    else:\n",
        "        return K.dot(x, kernel)\n",
        "\n",
        "class AttentionWithContext(Layer):\n",
        "    def __init__(self,\n",
        "                 W_regularizer=None, u_regularizer=None, b_regularizer=None,\n",
        "                 W_constraint=None, u_constraint=None, b_constraint=None,\n",
        "                 bias=True, **kwargs): \n",
        "        self.supports_masking = True\n",
        "        self.init = initializers.get('glorot_uniform') \n",
        "        self.W_regularizer = regularizers.get(W_regularizer)\n",
        "        self.u_regularizer = regularizers.get(u_regularizer)\n",
        "        self.b_regularizer = regularizers.get(b_regularizer) \n",
        "        self.W_constraint = constraints.get(W_constraint)\n",
        "        self.u_constraint = constraints.get(u_constraint)\n",
        "        self.b_constraint = constraints.get(b_constraint) \n",
        "        self.bias = bias\n",
        "        super(AttentionWithContext, self).__init__(**kwargs)\n",
        " \n",
        "    def build(self, input_shape):\n",
        "        assert len(input_shape) == 3\n",
        "        self.W = self.add_weight(shape=(input_shape[-1], input_shape[-1],),\n",
        "                                 initializer=self.init,\n",
        "                                 name='{}_W'.format(self.name),\n",
        "                                 regularizer=self.W_regularizer,\n",
        "                                 constraint=self.W_constraint)\n",
        "        if self.bias:\n",
        "            self.b = self.add_weight(shape=(input_shape[-1],),\n",
        "                                     initializer='zero',\n",
        "                                     name='{}_b'.format(self.name),\n",
        "                                     regularizer=self.b_regularizer,\n",
        "                                     constraint=self.b_constraint) \n",
        "            self.u = self.add_weight(shape=(input_shape[-1],),\n",
        "                                 initializer=self.init,\n",
        "                                 name='{}_u'.format(self.name),\n",
        "                                 regularizer=self.u_regularizer,\n",
        "                                 constraint=self.u_constraint) \n",
        "        super(AttentionWithContext, self).build(input_shape)\n",
        " \n",
        "    def compute_mask(self, input, input_mask=None):\n",
        "        return None\n",
        " \n",
        "    def call(self, x, mask=None):\n",
        "        uit = dot_product(x, self.W) \n",
        "        if self.bias:\n",
        "            uit += self.b \n",
        "        uit = K.tanh(uit)\n",
        "        ait = dot_product(uit, self.u) \n",
        "        a = K.exp(ait)\n",
        "        if mask is not None:\n",
        "            a *= K.cast(mask, K.floatx())\n",
        "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) \n",
        "        a = K.expand_dims(a)\n",
        "        weighted_input = x * a\n",
        "        return K.sum(weighted_input, axis=1)\n",
        " \n",
        "    def compute_output_shape(self, input_shape):\n",
        "        return input_shape[0], input_shape[-1]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nz2Up_g3xNNO"
      },
      "source": [
        "main_input = Input(shape=(1000,12), dtype='float32', name='main_input')\n",
        "x = Convolution1D(12, 3, padding='same')(main_input)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 24, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 24, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 24, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 24, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 48, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "cnnout = Dropout(0.2)(x)\n",
        "x = Bidirectional(CuDNNGRU(12, input_shape=(32,12),return_sequences=True,return_state=False))(cnnout)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = AttentionWithContext()(x)\n",
        "x = BatchNormalization()(x)\n",
        "last_dense = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(last_dense)\n",
        "main_output = Dense(num_classes,activation='sigmoid')(x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dadm3EOPdeK5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d2ea8779-09af-4b59-aa22-0deeb77e61ca"
      },
      "source": [
        "model = Model(main_input,main_output)\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "main_input (InputLayer)      [(None, 1000, 12)]        0         \n",
            "_________________________________________________________________\n",
            "conv1d_15 (Conv1D)           (None, 1000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_17 (LeakyReLU)   (None, 1000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_16 (Conv1D)           (None, 1000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_18 (LeakyReLU)   (None, 1000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_17 (Conv1D)           (None, 500, 12)           3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_19 (LeakyReLU)   (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_7 (Dropout)          (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_18 (Conv1D)           (None, 500, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_20 (LeakyReLU)   (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_19 (Conv1D)           (None, 500, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_21 (LeakyReLU)   (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_20 (Conv1D)           (None, 250, 12)           3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_22 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_8 (Dropout)          (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_21 (Conv1D)           (None, 250, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_23 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_22 (Conv1D)           (None, 250, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_24 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_23 (Conv1D)           (None, 125, 12)           3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_25 (LeakyReLU)   (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_9 (Dropout)          (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_24 (Conv1D)           (None, 125, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_26 (LeakyReLU)   (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_25 (Conv1D)           (None, 125, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_27 (LeakyReLU)   (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_26 (Conv1D)           (None, 63, 12)            3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_28 (LeakyReLU)   (None, 63, 12)            0         \n",
            "_________________________________________________________________\n",
            "dropout_10 (Dropout)         (None, 63, 12)            0         \n",
            "_________________________________________________________________\n",
            "conv1d_27 (Conv1D)           (None, 63, 12)            444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_29 (LeakyReLU)   (None, 63, 12)            0         \n",
            "_________________________________________________________________\n",
            "conv1d_28 (Conv1D)           (None, 63, 12)            444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_30 (LeakyReLU)   (None, 63, 12)            0         \n",
            "_________________________________________________________________\n",
            "conv1d_29 (Conv1D)           (None, 32, 12)            6924      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_31 (LeakyReLU)   (None, 32, 12)            0         \n",
            "_________________________________________________________________\n",
            "dropout_11 (Dropout)         (None, 32, 12)            0         \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, 32, 24)            1872      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_32 (LeakyReLU)   (None, 32, 24)            0         \n",
            "_________________________________________________________________\n",
            "dropout_12 (Dropout)         (None, 32, 24)            0         \n",
            "_________________________________________________________________\n",
            "attention_with_context_1 (At (None, 24)                624       \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 24)                96        \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_33 (LeakyReLU)   (None, 24)                0         \n",
            "_________________________________________________________________\n",
            "dropout_13 (Dropout)         (None, 24)                0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 9)                 225       \n",
            "=================================================================\n",
            "Total params: 28,053\n",
            "Trainable params: 28,005\n",
            "Non-trainable params: 48\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pyZ1ncXtNb6Z",
        "outputId": "c7449da4-e4f6-45a6-e444-32feee15a3d4"
      },
      "source": [
        "intermediate_layer_model = Model(main_input,last_dense)\n",
        "\n",
        "intermediate_layer_model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "main_input (InputLayer)      [(None, 1000, 12)]        0         \n",
            "_________________________________________________________________\n",
            "conv1d_15 (Conv1D)           (None, 1000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_17 (LeakyReLU)   (None, 1000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_16 (Conv1D)           (None, 1000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_18 (LeakyReLU)   (None, 1000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_17 (Conv1D)           (None, 500, 12)           3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_19 (LeakyReLU)   (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_7 (Dropout)          (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_18 (Conv1D)           (None, 500, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_20 (LeakyReLU)   (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_19 (Conv1D)           (None, 500, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_21 (LeakyReLU)   (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_20 (Conv1D)           (None, 250, 12)           3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_22 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_8 (Dropout)          (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_21 (Conv1D)           (None, 250, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_23 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_22 (Conv1D)           (None, 250, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_24 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_23 (Conv1D)           (None, 125, 12)           3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_25 (LeakyReLU)   (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_9 (Dropout)          (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_24 (Conv1D)           (None, 125, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_26 (LeakyReLU)   (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_25 (Conv1D)           (None, 125, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_27 (LeakyReLU)   (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_26 (Conv1D)           (None, 63, 12)            3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_28 (LeakyReLU)   (None, 63, 12)            0         \n",
            "_________________________________________________________________\n",
            "dropout_10 (Dropout)         (None, 63, 12)            0         \n",
            "_________________________________________________________________\n",
            "conv1d_27 (Conv1D)           (None, 63, 12)            444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_29 (LeakyReLU)   (None, 63, 12)            0         \n",
            "_________________________________________________________________\n",
            "conv1d_28 (Conv1D)           (None, 63, 12)            444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_30 (LeakyReLU)   (None, 63, 12)            0         \n",
            "_________________________________________________________________\n",
            "conv1d_29 (Conv1D)           (None, 32, 12)            6924      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_31 (LeakyReLU)   (None, 32, 12)            0         \n",
            "_________________________________________________________________\n",
            "dropout_11 (Dropout)         (None, 32, 12)            0         \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, 32, 24)            1872      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_32 (LeakyReLU)   (None, 32, 24)            0         \n",
            "_________________________________________________________________\n",
            "dropout_12 (Dropout)         (None, 32, 24)            0         \n",
            "_________________________________________________________________\n",
            "attention_with_context_1 (At (None, 24)                624       \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 24)                96        \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_33 (LeakyReLU)   (None, 24)                0         \n",
            "=================================================================\n",
            "Total params: 27,828\n",
            "Trainable params: 27,780\n",
            "Non-trainable params: 48\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rxQ815UZZPf7"
      },
      "source": [
        "model.compile(optimizer=Adam(learning_rate=1e-4),\n",
        "                loss='binary_crossentropy',\n",
        "                metrics=['AUC'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DdpY5Zmcdlam",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ce4c361c-a5bc-46ac-e2db-280155345dbe"
      },
      "source": [
        "model.fit(X_tr, y_train, \n",
        "          validation_split=0.1,\n",
        "          epochs=20,\n",
        "          batch_size=128,\n",
        "          shuffle=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "44/44 [==============================] - 4s 24ms/step - loss: 0.1182 - auc: 0.9698 - val_loss: 0.1325 - val_auc: 0.9615\n",
            "Epoch 2/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1168 - auc: 0.9701 - val_loss: 0.1302 - val_auc: 0.9630\n",
            "Epoch 3/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1166 - auc: 0.9708 - val_loss: 0.1297 - val_auc: 0.9624\n",
            "Epoch 4/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1175 - auc: 0.9695 - val_loss: 0.1282 - val_auc: 0.9637\n",
            "Epoch 5/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1182 - auc: 0.9697 - val_loss: 0.1297 - val_auc: 0.9629\n",
            "Epoch 6/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1170 - auc: 0.9707 - val_loss: 0.1306 - val_auc: 0.9627\n",
            "Epoch 7/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1176 - auc: 0.9704 - val_loss: 0.1329 - val_auc: 0.9620\n",
            "Epoch 8/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1165 - auc: 0.9709 - val_loss: 0.1297 - val_auc: 0.9631\n",
            "Epoch 9/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1178 - auc: 0.9694 - val_loss: 0.1298 - val_auc: 0.9637\n",
            "Epoch 10/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1172 - auc: 0.9705 - val_loss: 0.1349 - val_auc: 0.9615\n",
            "Epoch 11/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1166 - auc: 0.9703 - val_loss: 0.1314 - val_auc: 0.9624\n",
            "Epoch 12/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1171 - auc: 0.9703 - val_loss: 0.1304 - val_auc: 0.9635\n",
            "Epoch 13/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1168 - auc: 0.9706 - val_loss: 0.1330 - val_auc: 0.9615\n",
            "Epoch 14/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1138 - auc: 0.9722 - val_loss: 0.1300 - val_auc: 0.9635\n",
            "Epoch 15/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1170 - auc: 0.9704 - val_loss: 0.1276 - val_auc: 0.9644\n",
            "Epoch 16/20\n",
            "44/44 [==============================] - 1s 19ms/step - loss: 0.1163 - auc: 0.9709 - val_loss: 0.1364 - val_auc: 0.9615\n",
            "Epoch 17/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1169 - auc: 0.9702 - val_loss: 0.1309 - val_auc: 0.9622\n",
            "Epoch 18/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1151 - auc: 0.9712 - val_loss: 0.1311 - val_auc: 0.9622\n",
            "Epoch 19/20\n",
            "44/44 [==============================] - 1s 20ms/step - loss: 0.1167 - auc: 0.9703 - val_loss: 0.1331 - val_auc: 0.9617\n",
            "Epoch 20/20\n",
            "44/44 [==============================] - 1s 19ms/step - loss: 0.1152 - auc: 0.9716 - val_loss: 0.1283 - val_auc: 0.9646\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7ffa7616e910>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TwU2yLuGGjRN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "outputId": "c78f348b-9353-40e7-c71b-a4dc6673c827"
      },
      "source": [
        "y_pred = model.predict(X_te)\n",
        "utils.evaluate_experiment(y_val, y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.945772</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0   0.945772"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eX0ZP_diokOv",
        "outputId": "02dbd40d-2c1f-453d-8f06-35ce9ff5047e"
      },
      "source": [
        "from sklearn.metrics import classification_report, fbeta_score, roc_auc_score, roc_curve, roc_curve, auc\n",
        "\n",
        "def find_optimal_cutoff_threshold(target, predicted):\n",
        "    \"\"\" \n",
        "    Find the optimal probability cutoff point for a classification model related to event rate\n",
        "    \"\"\"\n",
        "    fpr, tpr, threshold = roc_curve(target, predicted)\n",
        "    optimal_idx = np.argmax(tpr - fpr)\n",
        "    optimal_threshold = threshold[optimal_idx]\n",
        "    return optimal_threshold\n",
        "\n",
        "def find_optimal_cutoff_thresholds(y_true, y_pred):\n",
        "\treturn [find_optimal_cutoff_threshold(y_true[:,i], y_pred[:,i]) for i in range(y_true.shape[1])]\n",
        "\n",
        "thresholds =  find_optimal_cutoff_thresholds_for_Gbeta(y_val , y_pred)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/9 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "optimize thresholds with respect to G_beta\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 9/9 [00:07<00:00,  1.28it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b6aE4hJIokOy",
        "outputId": "aec0011b-928f-4c49-f239-09b4e78b01a3"
      },
      "source": [
        "y_pred_binary = apply_thresholds(y_pred, thresholds)\n",
        "y_pred_binary[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 0, 0, 0, 0, 0, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "mzA7Ye6sokO0",
        "outputId": "3eab5900-b231-48b7-e4e2-03abe2c2e468"
      },
      "source": [
        "utils.evaluate_experiment(y_val, y_pred_binary)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.886316</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0   0.886316"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rwHW3QRqokO1"
      },
      "source": [
        "from sklearn.metrics import (confusion_matrix,\n",
        "                             precision_score, recall_score, f1_score,\n",
        "                             precision_recall_curve, average_precision_score , balanced_accuracy_score)\n",
        "diagnosis = ['NORM', 'AFIB', '1AVB', 'CLBBB', 'CRBBB', 'PAC', 'VPC', 'STD_', 'STE_']\n",
        "def specificity_score(y_true, y_pred):\n",
        "    m = confusion_matrix(y_true, y_pred, labels=[0, 1])\n",
        "    spc = m[0, 0] * 1.0 / (m[0, 0] + m[0, 1])\n",
        "    return spc\n",
        "\n",
        "def get_scores(y_true, y_pred, score_fun):\n",
        "    nclasses = np.shape(y_true)[1]\n",
        "    scores = []\n",
        "    for name, fun in score_fun.items():\n",
        "        scores += [[fun(y_true[:, k], y_pred[:, k]) for k in range(nclasses)]]\n",
        "    return np.array(scores).T\n",
        "\n",
        "\n",
        "score_fun = {'Precision': precision_score,\n",
        "             'Recall': recall_score, 'Specificity': specificity_score,\n",
        "             'F1 score': f1_score , 'balanced accuracy': balanced_accuracy_score}\n",
        "\n",
        "scores_list = []\n",
        "for y_pred in [y_pred_binary]:\n",
        "    # Compute scores\n",
        "    scores = get_scores(y_val, y_pred, score_fun)\n",
        "    # Put them into a data frame\n",
        "    scores_df = pd.DataFrame(scores,index=diagnosis, columns=score_fun.keys())\n",
        "    # Append\n",
        "    scores_list.append(scores_df)\n",
        "# Concatenate dataframes\n",
        "scores_all_df = pd.concat(scores_list, axis=1, keys=[''])\n",
        "# Change multiindex levels\n",
        "scores_all_df = scores_all_df.swaplevel(0, 1, axis=1)\n",
        "scores_all_df = scores_all_df.reindex(level=0, columns=score_fun.keys())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        },
        "id": "5u2FpnyxokO3",
        "outputId": "cace12e4-9aa3-413c-d2ae-2657be26bb03"
      },
      "source": [
        "#Gbeta thershold\n",
        "scores_all_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Specificity</th>\n",
              "      <th>F1 score</th>\n",
              "      <th>balanced accuracy</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>NORM</th>\n",
              "      <td>0.846154</td>\n",
              "      <td>0.916667</td>\n",
              "      <td>0.980583</td>\n",
              "      <td>0.880000</td>\n",
              "      <td>0.948625</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AFIB</th>\n",
              "      <td>0.896825</td>\n",
              "      <td>0.918699</td>\n",
              "      <td>0.977072</td>\n",
              "      <td>0.907631</td>\n",
              "      <td>0.947886</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1AVB</th>\n",
              "      <td>0.863636</td>\n",
              "      <td>0.791667</td>\n",
              "      <td>0.995495</td>\n",
              "      <td>0.826087</td>\n",
              "      <td>0.893581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CLBBB</th>\n",
              "      <td>0.901478</td>\n",
              "      <td>0.948187</td>\n",
              "      <td>0.959759</td>\n",
              "      <td>0.924242</td>\n",
              "      <td>0.953973</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CRBBB</th>\n",
              "      <td>0.754902</td>\n",
              "      <td>0.836957</td>\n",
              "      <td>0.958194</td>\n",
              "      <td>0.793814</td>\n",
              "      <td>0.897575</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PAC</th>\n",
              "      <td>0.686275</td>\n",
              "      <td>0.573770</td>\n",
              "      <td>0.974563</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.774167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>VPC</th>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.862069</td>\n",
              "      <td>0.958541</td>\n",
              "      <td>0.802139</td>\n",
              "      <td>0.910305</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>STD_</th>\n",
              "      <td>0.666667</td>\n",
              "      <td>0.636364</td>\n",
              "      <td>0.989521</td>\n",
              "      <td>0.651163</td>\n",
              "      <td>0.812942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>STE_</th>\n",
              "      <td>0.675676</td>\n",
              "      <td>0.714286</td>\n",
              "      <td>0.961290</td>\n",
              "      <td>0.694444</td>\n",
              "      <td>0.837788</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      Precision    Recall Specificity  F1 score balanced accuracy\n",
              "                                                                 \n",
              "NORM   0.846154  0.916667    0.980583  0.880000          0.948625\n",
              "AFIB   0.896825  0.918699    0.977072  0.907631          0.947886\n",
              "1AVB   0.863636  0.791667    0.995495  0.826087          0.893581\n",
              "CLBBB  0.901478  0.948187    0.959759  0.924242          0.953973\n",
              "CRBBB  0.754902  0.836957    0.958194  0.793814          0.897575\n",
              "PAC    0.686275  0.573770    0.974563  0.625000          0.774167\n",
              "VPC    0.750000  0.862069    0.958541  0.802139          0.910305\n",
              "STD_   0.666667  0.636364    0.989521  0.651163          0.812942\n",
              "STE_   0.675676  0.714286    0.961290  0.694444          0.837788"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 644
        },
        "id": "XEEBTihPokO4",
        "outputId": "aeb411d7-5ce5-4ea8-e9cf-54636a243078"
      },
      "source": [
        "# %% Confusion matrices (Supplementary Table 1)\n",
        "from sklearn.metrics import (confusion_matrix,\n",
        "                             precision_score, recall_score, f1_score,\n",
        "                             precision_recall_curve, average_precision_score)\n",
        "\n",
        "import xarray as xr\n",
        "\n",
        "M = [[confusion_matrix(y_val[:, k], y_pred[:, k], labels=[0, 1])\n",
        "      for k in range(9)] for y_pred in [y_pred_binary]]\n",
        "\n",
        "M_xarray = xr.DataArray(np.array(M),\n",
        "                        dims=['predictor', 'diagnosis', 'true label', 'predicted label'],\n",
        "                        coords={'predictor': ['CNN+RNN'],\n",
        "                                'diagnosis': diagnosis,\n",
        "                                'true label': ['not present', 'present'],\n",
        "                                'predicted label': ['not present', 'present']})\n",
        "confusion_matrices = M_xarray.to_dataframe('n')\n",
        "confusion_matrices = confusion_matrices.reorder_levels([1, 2, 3, 0], axis=0)\n",
        "confusion_matrices = confusion_matrices.unstack()\n",
        "confusion_matrices = confusion_matrices.unstack()\n",
        "confusion_matrices = confusion_matrices['n']\n",
        "confusion_matrices"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>predictor</th>\n",
              "      <th colspan=\"2\" halign=\"left\">CNN+RNN</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>predicted label</th>\n",
              "      <th>not present</th>\n",
              "      <th>present</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>diagnosis</th>\n",
              "      <th>true label</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">NORM</th>\n",
              "      <th>not present</th>\n",
              "      <td>606</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>6</td>\n",
              "      <td>66</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">AFIB</th>\n",
              "      <th>not present</th>\n",
              "      <td>554</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>10</td>\n",
              "      <td>113</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1AVB</th>\n",
              "      <th>not present</th>\n",
              "      <td>663</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>5</td>\n",
              "      <td>19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">CLBBB</th>\n",
              "      <th>not present</th>\n",
              "      <td>477</td>\n",
              "      <td>20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>10</td>\n",
              "      <td>183</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">CRBBB</th>\n",
              "      <th>not present</th>\n",
              "      <td>573</td>\n",
              "      <td>25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>15</td>\n",
              "      <td>77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">PAC</th>\n",
              "      <th>not present</th>\n",
              "      <td>613</td>\n",
              "      <td>16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>26</td>\n",
              "      <td>35</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">VPC</th>\n",
              "      <th>not present</th>\n",
              "      <td>578</td>\n",
              "      <td>25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>12</td>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">STD_</th>\n",
              "      <th>not present</th>\n",
              "      <td>661</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>8</td>\n",
              "      <td>14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">STE_</th>\n",
              "      <th>not present</th>\n",
              "      <td>596</td>\n",
              "      <td>24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "predictor                 CNN+RNN        \n",
              "predicted label       not present present\n",
              "diagnosis true label                     \n",
              "NORM      not present         606      12\n",
              "          present               6      66\n",
              "AFIB      not present         554      13\n",
              "          present              10     113\n",
              "1AVB      not present         663       3\n",
              "          present               5      19\n",
              "CLBBB     not present         477      20\n",
              "          present              10     183\n",
              "CRBBB     not present         573      25\n",
              "          present              15      77\n",
              "PAC       not present         613      16\n",
              "          present              26      35\n",
              "VPC       not present         578      25\n",
              "          present              12      75\n",
              "STD_      not present         661       7\n",
              "          present               8      14\n",
              "STE_      not present         596      24\n",
              "          present              20      50"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HM7fLhJhOio_"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "D0Q9ZyKqP3eA",
        "outputId": "3eb60c99-8910-405d-98f5-966ea4e53c68"
      },
      "source": [
        "intermediate_output = intermediate_layer_model.predict(X_tr) \n",
        "intermediate_output = pd.DataFrame(data=intermediate_output)\n",
        "intermediate_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3.001643</td>\n",
              "      <td>-0.067764</td>\n",
              "      <td>-0.288871</td>\n",
              "      <td>-0.090582</td>\n",
              "      <td>2.411433</td>\n",
              "      <td>1.748587</td>\n",
              "      <td>-0.402312</td>\n",
              "      <td>0.430912</td>\n",
              "      <td>-0.392957</td>\n",
              "      <td>-0.244924</td>\n",
              "      <td>0.781219</td>\n",
              "      <td>1.680753</td>\n",
              "      <td>-0.359231</td>\n",
              "      <td>2.324959</td>\n",
              "      <td>-0.270377</td>\n",
              "      <td>1.821432</td>\n",
              "      <td>2.283545</td>\n",
              "      <td>-0.327521</td>\n",
              "      <td>-0.127641</td>\n",
              "      <td>-0.233211</td>\n",
              "      <td>-0.170167</td>\n",
              "      <td>2.150958</td>\n",
              "      <td>1.642843</td>\n",
              "      <td>2.255541</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.944204</td>\n",
              "      <td>2.203956</td>\n",
              "      <td>2.494365</td>\n",
              "      <td>-0.193275</td>\n",
              "      <td>-0.153300</td>\n",
              "      <td>1.267270</td>\n",
              "      <td>1.886377</td>\n",
              "      <td>-0.088683</td>\n",
              "      <td>1.735864</td>\n",
              "      <td>1.757166</td>\n",
              "      <td>2.951988</td>\n",
              "      <td>-0.095238</td>\n",
              "      <td>-0.255146</td>\n",
              "      <td>2.853224</td>\n",
              "      <td>-0.059378</td>\n",
              "      <td>-0.341435</td>\n",
              "      <td>-0.354121</td>\n",
              "      <td>3.077388</td>\n",
              "      <td>-0.179307</td>\n",
              "      <td>0.016734</td>\n",
              "      <td>-0.251803</td>\n",
              "      <td>-0.092571</td>\n",
              "      <td>-0.228424</td>\n",
              "      <td>-0.242189</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-0.219171</td>\n",
              "      <td>-0.185172</td>\n",
              "      <td>-0.272619</td>\n",
              "      <td>-0.056880</td>\n",
              "      <td>-0.170273</td>\n",
              "      <td>-0.422777</td>\n",
              "      <td>1.851090</td>\n",
              "      <td>-0.087204</td>\n",
              "      <td>1.447175</td>\n",
              "      <td>-0.321971</td>\n",
              "      <td>-0.343484</td>\n",
              "      <td>2.818882</td>\n",
              "      <td>-0.302385</td>\n",
              "      <td>-0.214753</td>\n",
              "      <td>3.071851</td>\n",
              "      <td>1.483443</td>\n",
              "      <td>-0.293914</td>\n",
              "      <td>-0.011953</td>\n",
              "      <td>3.601426</td>\n",
              "      <td>1.099658</td>\n",
              "      <td>3.204190</td>\n",
              "      <td>-0.212399</td>\n",
              "      <td>0.917357</td>\n",
              "      <td>1.635759</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.172477</td>\n",
              "      <td>-0.198357</td>\n",
              "      <td>2.611312</td>\n",
              "      <td>2.086379</td>\n",
              "      <td>-0.173512</td>\n",
              "      <td>1.203426</td>\n",
              "      <td>1.072673</td>\n",
              "      <td>-0.200906</td>\n",
              "      <td>0.458440</td>\n",
              "      <td>1.297238</td>\n",
              "      <td>0.528037</td>\n",
              "      <td>-0.090203</td>\n",
              "      <td>1.569444</td>\n",
              "      <td>-0.074765</td>\n",
              "      <td>0.734828</td>\n",
              "      <td>1.426463</td>\n",
              "      <td>-0.312277</td>\n",
              "      <td>-0.086207</td>\n",
              "      <td>-0.094051</td>\n",
              "      <td>-0.063027</td>\n",
              "      <td>-0.084447</td>\n",
              "      <td>0.493436</td>\n",
              "      <td>0.631761</td>\n",
              "      <td>0.070878</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3.224120</td>\n",
              "      <td>-0.010500</td>\n",
              "      <td>-0.257060</td>\n",
              "      <td>-0.105387</td>\n",
              "      <td>1.687323</td>\n",
              "      <td>2.732973</td>\n",
              "      <td>-0.403518</td>\n",
              "      <td>0.308186</td>\n",
              "      <td>-0.388403</td>\n",
              "      <td>-0.154453</td>\n",
              "      <td>1.839732</td>\n",
              "      <td>1.963377</td>\n",
              "      <td>-0.375995</td>\n",
              "      <td>2.495319</td>\n",
              "      <td>-0.280214</td>\n",
              "      <td>1.866578</td>\n",
              "      <td>2.273375</td>\n",
              "      <td>-0.305717</td>\n",
              "      <td>-0.185327</td>\n",
              "      <td>-0.244148</td>\n",
              "      <td>-0.233412</td>\n",
              "      <td>2.302249</td>\n",
              "      <td>1.115411</td>\n",
              "      <td>2.182754</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6182</th>\n",
              "      <td>0.523405</td>\n",
              "      <td>1.813215</td>\n",
              "      <td>-0.289492</td>\n",
              "      <td>-0.043818</td>\n",
              "      <td>2.340290</td>\n",
              "      <td>-0.275741</td>\n",
              "      <td>-0.310848</td>\n",
              "      <td>2.998348</td>\n",
              "      <td>-0.023842</td>\n",
              "      <td>0.220957</td>\n",
              "      <td>-0.072603</td>\n",
              "      <td>-0.037156</td>\n",
              "      <td>-0.012081</td>\n",
              "      <td>2.134110</td>\n",
              "      <td>-0.032463</td>\n",
              "      <td>-0.057847</td>\n",
              "      <td>0.369818</td>\n",
              "      <td>-0.115334</td>\n",
              "      <td>0.503904</td>\n",
              "      <td>-0.070384</td>\n",
              "      <td>-0.095565</td>\n",
              "      <td>0.484909</td>\n",
              "      <td>2.325752</td>\n",
              "      <td>-0.109211</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6183</th>\n",
              "      <td>3.402399</td>\n",
              "      <td>1.001953</td>\n",
              "      <td>2.576252</td>\n",
              "      <td>0.859229</td>\n",
              "      <td>0.017726</td>\n",
              "      <td>-0.262102</td>\n",
              "      <td>1.462629</td>\n",
              "      <td>-0.101996</td>\n",
              "      <td>-0.241797</td>\n",
              "      <td>1.657388</td>\n",
              "      <td>-0.087862</td>\n",
              "      <td>-0.189367</td>\n",
              "      <td>-0.046283</td>\n",
              "      <td>-0.186677</td>\n",
              "      <td>0.250819</td>\n",
              "      <td>-0.377086</td>\n",
              "      <td>2.195347</td>\n",
              "      <td>1.497748</td>\n",
              "      <td>2.627158</td>\n",
              "      <td>2.663933</td>\n",
              "      <td>0.268404</td>\n",
              "      <td>-0.083794</td>\n",
              "      <td>-0.335468</td>\n",
              "      <td>0.283352</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6184</th>\n",
              "      <td>-0.314032</td>\n",
              "      <td>-0.133376</td>\n",
              "      <td>-0.437824</td>\n",
              "      <td>1.919265</td>\n",
              "      <td>-0.353851</td>\n",
              "      <td>2.770180</td>\n",
              "      <td>-0.164570</td>\n",
              "      <td>4.375890</td>\n",
              "      <td>1.988236</td>\n",
              "      <td>-0.193792</td>\n",
              "      <td>-0.136241</td>\n",
              "      <td>3.281760</td>\n",
              "      <td>-0.132586</td>\n",
              "      <td>-0.291921</td>\n",
              "      <td>1.406449</td>\n",
              "      <td>-0.193715</td>\n",
              "      <td>1.293892</td>\n",
              "      <td>-0.046822</td>\n",
              "      <td>-0.213633</td>\n",
              "      <td>4.084955</td>\n",
              "      <td>-0.027310</td>\n",
              "      <td>-0.511739</td>\n",
              "      <td>-0.513955</td>\n",
              "      <td>-0.316522</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6185</th>\n",
              "      <td>-0.213868</td>\n",
              "      <td>-0.202970</td>\n",
              "      <td>-0.385915</td>\n",
              "      <td>-0.144116</td>\n",
              "      <td>-0.066047</td>\n",
              "      <td>-0.479793</td>\n",
              "      <td>1.624040</td>\n",
              "      <td>-0.035350</td>\n",
              "      <td>1.320960</td>\n",
              "      <td>-0.356047</td>\n",
              "      <td>-0.365784</td>\n",
              "      <td>3.151830</td>\n",
              "      <td>-0.308871</td>\n",
              "      <td>-0.157329</td>\n",
              "      <td>2.828130</td>\n",
              "      <td>1.744192</td>\n",
              "      <td>-0.414381</td>\n",
              "      <td>-0.137230</td>\n",
              "      <td>3.320789</td>\n",
              "      <td>0.798270</td>\n",
              "      <td>3.055030</td>\n",
              "      <td>-0.157947</td>\n",
              "      <td>2.769519</td>\n",
              "      <td>2.179981</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6186</th>\n",
              "      <td>1.741241</td>\n",
              "      <td>0.189679</td>\n",
              "      <td>2.212897</td>\n",
              "      <td>0.261870</td>\n",
              "      <td>-0.036305</td>\n",
              "      <td>-0.251215</td>\n",
              "      <td>1.369825</td>\n",
              "      <td>-0.083516</td>\n",
              "      <td>-0.087509</td>\n",
              "      <td>1.673689</td>\n",
              "      <td>-0.177786</td>\n",
              "      <td>-0.156941</td>\n",
              "      <td>-0.016572</td>\n",
              "      <td>-0.139370</td>\n",
              "      <td>0.049949</td>\n",
              "      <td>-0.240419</td>\n",
              "      <td>1.302243</td>\n",
              "      <td>0.153258</td>\n",
              "      <td>1.904730</td>\n",
              "      <td>1.851887</td>\n",
              "      <td>0.464386</td>\n",
              "      <td>0.508923</td>\n",
              "      <td>-0.244415</td>\n",
              "      <td>0.717278</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6187 rows × 24 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            0         1         2   ...        21        22        23\n",
              "0     3.001643 -0.067764 -0.288871  ...  2.150958  1.642843  2.255541\n",
              "1     0.944204  2.203956  2.494365  ... -0.092571 -0.228424 -0.242189\n",
              "2    -0.219171 -0.185172 -0.272619  ... -0.212399  0.917357  1.635759\n",
              "3    -0.172477 -0.198357  2.611312  ...  0.493436  0.631761  0.070878\n",
              "4     3.224120 -0.010500 -0.257060  ...  2.302249  1.115411  2.182754\n",
              "...        ...       ...       ...  ...       ...       ...       ...\n",
              "6182  0.523405  1.813215 -0.289492  ...  0.484909  2.325752 -0.109211\n",
              "6183  3.402399  1.001953  2.576252  ... -0.083794 -0.335468  0.283352\n",
              "6184 -0.314032 -0.133376 -0.437824  ... -0.511739 -0.513955 -0.316522\n",
              "6185 -0.213868 -0.202970 -0.385915  ... -0.157947  2.769519  2.179981\n",
              "6186  1.741241  0.189679  2.212897  ...  0.508923 -0.244415  0.717278\n",
              "\n",
              "[6187 rows x 24 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hh1zRjuCSU5b",
        "outputId": "03ecde46-84d7-42ef-ff4e-9f60fccc53fc"
      },
      "source": [
        "intermediate_output.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(6187, 24)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "zi9-dauYVQfm",
        "outputId": "0c108c32-a9b7-4ea9-c734-0bad66c68b58"
      },
      "source": [
        "# submission_cnn = incep.predict(X_val)\n",
        "intermediate_test_output = intermediate_layer_model.predict(X_te)\n",
        "intermediate_test_output = pd.DataFrame(data=intermediate_test_output)\n",
        "intermediate_test_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.255392</td>\n",
              "      <td>-0.211745</td>\n",
              "      <td>-0.406428</td>\n",
              "      <td>-0.142480</td>\n",
              "      <td>-0.125017</td>\n",
              "      <td>-0.489880</td>\n",
              "      <td>1.848378</td>\n",
              "      <td>0.334954</td>\n",
              "      <td>1.612910</td>\n",
              "      <td>-0.359512</td>\n",
              "      <td>-0.393796</td>\n",
              "      <td>3.109035</td>\n",
              "      <td>-0.252718</td>\n",
              "      <td>-0.182750</td>\n",
              "      <td>3.527593</td>\n",
              "      <td>1.619462</td>\n",
              "      <td>-0.462926</td>\n",
              "      <td>-0.075328</td>\n",
              "      <td>3.171779</td>\n",
              "      <td>0.617038</td>\n",
              "      <td>3.235300</td>\n",
              "      <td>-0.294856</td>\n",
              "      <td>2.669544</td>\n",
              "      <td>1.866967</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.525714</td>\n",
              "      <td>0.062044</td>\n",
              "      <td>1.557777</td>\n",
              "      <td>0.059896</td>\n",
              "      <td>-0.107384</td>\n",
              "      <td>-0.300601</td>\n",
              "      <td>1.515254</td>\n",
              "      <td>-0.036957</td>\n",
              "      <td>0.816227</td>\n",
              "      <td>1.588835</td>\n",
              "      <td>-0.238807</td>\n",
              "      <td>-0.103106</td>\n",
              "      <td>-0.082493</td>\n",
              "      <td>-0.138587</td>\n",
              "      <td>0.712973</td>\n",
              "      <td>-0.222239</td>\n",
              "      <td>0.889631</td>\n",
              "      <td>0.847310</td>\n",
              "      <td>1.873084</td>\n",
              "      <td>1.778764</td>\n",
              "      <td>0.631984</td>\n",
              "      <td>0.163307</td>\n",
              "      <td>-0.215947</td>\n",
              "      <td>0.368411</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.285223</td>\n",
              "      <td>2.865669</td>\n",
              "      <td>2.576134</td>\n",
              "      <td>-0.257837</td>\n",
              "      <td>-0.213428</td>\n",
              "      <td>2.624957</td>\n",
              "      <td>1.912687</td>\n",
              "      <td>-0.128523</td>\n",
              "      <td>1.846269</td>\n",
              "      <td>1.780268</td>\n",
              "      <td>4.874685</td>\n",
              "      <td>-0.073190</td>\n",
              "      <td>-0.312853</td>\n",
              "      <td>2.891600</td>\n",
              "      <td>-0.153347</td>\n",
              "      <td>-0.203334</td>\n",
              "      <td>-0.399434</td>\n",
              "      <td>2.994854</td>\n",
              "      <td>-0.207030</td>\n",
              "      <td>-0.014133</td>\n",
              "      <td>-0.287892</td>\n",
              "      <td>-0.058565</td>\n",
              "      <td>-0.339682</td>\n",
              "      <td>-0.276051</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.273353</td>\n",
              "      <td>0.120967</td>\n",
              "      <td>-0.437371</td>\n",
              "      <td>1.034279</td>\n",
              "      <td>-0.311853</td>\n",
              "      <td>2.482226</td>\n",
              "      <td>-0.168039</td>\n",
              "      <td>3.370830</td>\n",
              "      <td>1.743863</td>\n",
              "      <td>-0.144708</td>\n",
              "      <td>0.694945</td>\n",
              "      <td>2.071692</td>\n",
              "      <td>-0.119326</td>\n",
              "      <td>-0.290215</td>\n",
              "      <td>1.416375</td>\n",
              "      <td>0.140978</td>\n",
              "      <td>2.513539</td>\n",
              "      <td>-0.045433</td>\n",
              "      <td>-0.206044</td>\n",
              "      <td>3.490529</td>\n",
              "      <td>-0.226120</td>\n",
              "      <td>-0.478638</td>\n",
              "      <td>-0.496032</td>\n",
              "      <td>-0.333505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.369120</td>\n",
              "      <td>3.283067</td>\n",
              "      <td>-0.244850</td>\n",
              "      <td>-0.201509</td>\n",
              "      <td>-0.143654</td>\n",
              "      <td>2.830542</td>\n",
              "      <td>0.478915</td>\n",
              "      <td>0.732054</td>\n",
              "      <td>1.763713</td>\n",
              "      <td>1.649326</td>\n",
              "      <td>2.323838</td>\n",
              "      <td>-0.063304</td>\n",
              "      <td>-0.283985</td>\n",
              "      <td>1.099096</td>\n",
              "      <td>-0.151460</td>\n",
              "      <td>1.009237</td>\n",
              "      <td>-0.247494</td>\n",
              "      <td>1.205474</td>\n",
              "      <td>-0.204992</td>\n",
              "      <td>0.569872</td>\n",
              "      <td>-0.281708</td>\n",
              "      <td>0.346507</td>\n",
              "      <td>-0.329852</td>\n",
              "      <td>-0.255727</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>685</th>\n",
              "      <td>2.074163</td>\n",
              "      <td>-0.020272</td>\n",
              "      <td>-0.282366</td>\n",
              "      <td>-0.064773</td>\n",
              "      <td>1.562740</td>\n",
              "      <td>2.176630</td>\n",
              "      <td>-0.400651</td>\n",
              "      <td>0.538170</td>\n",
              "      <td>-0.397890</td>\n",
              "      <td>-0.269159</td>\n",
              "      <td>1.730328</td>\n",
              "      <td>1.151343</td>\n",
              "      <td>-0.287263</td>\n",
              "      <td>2.106034</td>\n",
              "      <td>-0.280524</td>\n",
              "      <td>1.900683</td>\n",
              "      <td>1.505880</td>\n",
              "      <td>-0.343726</td>\n",
              "      <td>-0.165426</td>\n",
              "      <td>-0.243654</td>\n",
              "      <td>-0.246007</td>\n",
              "      <td>3.098737</td>\n",
              "      <td>1.659626</td>\n",
              "      <td>2.171075</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>686</th>\n",
              "      <td>3.849508</td>\n",
              "      <td>1.430745</td>\n",
              "      <td>2.997154</td>\n",
              "      <td>1.051703</td>\n",
              "      <td>-0.030274</td>\n",
              "      <td>-0.278380</td>\n",
              "      <td>1.630284</td>\n",
              "      <td>-0.172496</td>\n",
              "      <td>-0.230444</td>\n",
              "      <td>1.733097</td>\n",
              "      <td>-0.098901</td>\n",
              "      <td>-0.208783</td>\n",
              "      <td>-0.040841</td>\n",
              "      <td>-0.207346</td>\n",
              "      <td>0.692708</td>\n",
              "      <td>-0.428068</td>\n",
              "      <td>2.437585</td>\n",
              "      <td>2.315858</td>\n",
              "      <td>2.813515</td>\n",
              "      <td>3.012163</td>\n",
              "      <td>0.089091</td>\n",
              "      <td>-0.162041</td>\n",
              "      <td>-0.395679</td>\n",
              "      <td>-0.040044</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>687</th>\n",
              "      <td>1.831842</td>\n",
              "      <td>-0.037180</td>\n",
              "      <td>-0.316673</td>\n",
              "      <td>-0.055374</td>\n",
              "      <td>1.922740</td>\n",
              "      <td>1.058200</td>\n",
              "      <td>-0.399168</td>\n",
              "      <td>0.897470</td>\n",
              "      <td>-0.379024</td>\n",
              "      <td>-0.285368</td>\n",
              "      <td>0.560102</td>\n",
              "      <td>1.390820</td>\n",
              "      <td>-0.314774</td>\n",
              "      <td>1.551067</td>\n",
              "      <td>-0.247142</td>\n",
              "      <td>1.790096</td>\n",
              "      <td>1.873255</td>\n",
              "      <td>-0.309355</td>\n",
              "      <td>-0.084726</td>\n",
              "      <td>-0.191027</td>\n",
              "      <td>-0.127434</td>\n",
              "      <td>1.644878</td>\n",
              "      <td>2.002734</td>\n",
              "      <td>2.035933</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>688</th>\n",
              "      <td>1.418620</td>\n",
              "      <td>1.312839</td>\n",
              "      <td>0.890478</td>\n",
              "      <td>-0.092359</td>\n",
              "      <td>-0.009147</td>\n",
              "      <td>1.888826</td>\n",
              "      <td>-0.027134</td>\n",
              "      <td>0.092668</td>\n",
              "      <td>0.727996</td>\n",
              "      <td>1.634679</td>\n",
              "      <td>1.880245</td>\n",
              "      <td>0.228764</td>\n",
              "      <td>-0.288326</td>\n",
              "      <td>1.956409</td>\n",
              "      <td>-0.159836</td>\n",
              "      <td>0.690284</td>\n",
              "      <td>0.083759</td>\n",
              "      <td>0.071297</td>\n",
              "      <td>-0.164456</td>\n",
              "      <td>-0.050125</td>\n",
              "      <td>-0.197531</td>\n",
              "      <td>0.608387</td>\n",
              "      <td>-0.101953</td>\n",
              "      <td>0.002561</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>689</th>\n",
              "      <td>3.939913</td>\n",
              "      <td>1.287440</td>\n",
              "      <td>3.115202</td>\n",
              "      <td>1.083256</td>\n",
              "      <td>-0.016072</td>\n",
              "      <td>-0.224370</td>\n",
              "      <td>1.592511</td>\n",
              "      <td>-0.202171</td>\n",
              "      <td>-0.254116</td>\n",
              "      <td>1.745371</td>\n",
              "      <td>-0.016542</td>\n",
              "      <td>-0.216851</td>\n",
              "      <td>-0.081072</td>\n",
              "      <td>-0.184292</td>\n",
              "      <td>0.094231</td>\n",
              "      <td>-0.410434</td>\n",
              "      <td>2.233528</td>\n",
              "      <td>1.939652</td>\n",
              "      <td>2.409657</td>\n",
              "      <td>2.807899</td>\n",
              "      <td>0.032683</td>\n",
              "      <td>-0.107119</td>\n",
              "      <td>-0.387117</td>\n",
              "      <td>0.197092</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>690 rows × 24 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           0         1         2   ...        21        22        23\n",
              "0   -0.255392 -0.211745 -0.406428  ... -0.294856  2.669544  1.866967\n",
              "1    0.525714  0.062044  1.557777  ...  0.163307 -0.215947  0.368411\n",
              "2    1.285223  2.865669  2.576134  ... -0.058565 -0.339682 -0.276051\n",
              "3   -0.273353  0.120967 -0.437371  ... -0.478638 -0.496032 -0.333505\n",
              "4    0.369120  3.283067 -0.244850  ...  0.346507 -0.329852 -0.255727\n",
              "..        ...       ...       ...  ...       ...       ...       ...\n",
              "685  2.074163 -0.020272 -0.282366  ...  3.098737  1.659626  2.171075\n",
              "686  3.849508  1.430745  2.997154  ... -0.162041 -0.395679 -0.040044\n",
              "687  1.831842 -0.037180 -0.316673  ...  1.644878  2.002734  2.035933\n",
              "688  1.418620  1.312839  0.890478  ...  0.608387 -0.101953  0.002561\n",
              "689  3.939913  1.287440  3.115202  ... -0.107119 -0.387117  0.197092\n",
              "\n",
              "[690 rows x 24 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YWi5cKXoasUs",
        "outputId": "aff37930-6e63-4a4c-b09a-1f7a7829f03d"
      },
      "source": [
        "import xgboost as xgb\n",
        "from sklearn.datasets import make_multilabel_classification\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.multioutput import MultiOutputClassifier\n",
        "from sklearn.metrics import accuracy_score , auc\n",
        "\n",
        "# create XGBoost instance with default hyper-parameters\n",
        "# xgb_estimator = xgb.XGBClassifier(objective='binary:logistic' , max_depth=3 , n_estimators=100 ,colsample_bytree=0.7 , )\n",
        "xgb_estimator = xgb.XGBClassifier(objective='binary:logistic' , max_depth=15 , n_estimators=200  , learning_rate=0.01, subsample=0.7\n",
        "                                  , gamma=0.8 , )\n",
        "\n",
        "# create MultiOutputClassifier instance with XGBoost model inside\n",
        "multilabel_model = MultiOutputClassifier(xgb_estimator)\n",
        "\n",
        "# fit the model\n",
        "multilabel_model.fit(intermediate_output, y_train)\n",
        "\n",
        "# evaluate on test data\n",
        "print('Accuracy on test data: {:.1f}%'.format(accuracy_score(y_val, multilabel_model.predict(intermediate_test_output))*100))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy on test data: 75.2%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "pg9V1XTffrwv",
        "outputId": "c20dd8f4-2314-4ca6-be58-d245cf73bf05"
      },
      "source": [
        "evaluate_experiment(y_val, multilabel_model.predict(intermediate_test_output))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.854471</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0   0.854471"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "9QqEp5oxtn9O",
        "outputId": "a55de258-a614-4444-9a50-92e7adc35994"
      },
      "source": [
        "evaluate_experiment(y_train, multilabel_model.predict(intermediate_output))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.925169</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0   0.925169"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHDJxyS1m60k"
      },
      "source": [
        "thresholds =  find_optimal_cutoff_thresholds(y_val , multilabel_model.predict(intermediate_test_output))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OJA9usx_m60k",
        "outputId": "7b7adfe4-9e1e-4853-c2c8-2e340e63d695"
      },
      "source": [
        "y_pred_binary = apply_thresholds(multilabel_model.predict(intermediate_test_output), thresholds)\n",
        "y_pred_binary[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 0, 0, 0, 0, 0, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        },
        "id": "YcIjZHzkmrdN",
        "outputId": "a5e6d946-1ffd-49d5-9844-8b4a09c2eeb6"
      },
      "source": [
        "from sklearn.metrics import (confusion_matrix,\n",
        "                             precision_score, recall_score, f1_score,\n",
        "                             precision_recall_curve, average_precision_score , balanced_accuracy_score ,roc_auc_score)\n",
        "diagnosis = ['NORM', 'AFIB', '1AVB', 'CLBBB', 'CRBBB', 'PAC', 'VPC', 'STD_', 'STE_']\n",
        "\n",
        "def specificity_score(y_true, y_pred):\n",
        "    m = confusion_matrix(y_true, y_pred, labels=[0, 1])\n",
        "    spc = m[0, 0] * 1.0 / (m[0, 0] + m[0, 1])\n",
        "    return spc\n",
        "\n",
        "def get_scores(y_true, y_pred, score_fun):\n",
        "    nclasses = np.shape(y_true)[1]\n",
        "    scores = []\n",
        "    for name, fun in score_fun.items():\n",
        "        scores += [[fun(y_true[:, k], y_pred[:, k]) for k in range(nclasses)]]\n",
        "    return np.array(scores).T\n",
        "\n",
        "\n",
        "score_fun = {'Precision': precision_score,\n",
        "             'Recall': recall_score, 'Specificity': specificity_score,\n",
        "             'F1 score': f1_score , 'balanced_accuracy_score': balanced_accuracy_score}\n",
        "\n",
        "scores_list = []\n",
        "for y_pred in [y_pred_binary]:\n",
        "    # Compute scores\n",
        "    scores = get_scores(y_val, y_pred, score_fun)\n",
        "    # Put them into a data frame\n",
        "    scores_df = pd.DataFrame(scores, index=diagnosis, columns=score_fun.keys())\n",
        "    # Append\n",
        "    scores_list.append(scores_df)\n",
        "# Concatenate dataframes\n",
        "scores_all_df = pd.concat(scores_list, axis=1, keys=[''])\n",
        "# Change multiindex levels\n",
        "scores_all_df = scores_all_df.swaplevel(0, 1, axis=1)\n",
        "scores_all_df = scores_all_df.reindex(level=0, columns=score_fun.keys())\n",
        "scores_all_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Specificity</th>\n",
              "      <th>F1 score</th>\n",
              "      <th>balanced_accuracy_score</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>NORM</th>\n",
              "      <td>0.615385</td>\n",
              "      <td>0.888889</td>\n",
              "      <td>0.935275</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.912082</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AFIB</th>\n",
              "      <td>0.862903</td>\n",
              "      <td>0.869919</td>\n",
              "      <td>0.970018</td>\n",
              "      <td>0.866397</td>\n",
              "      <td>0.919968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1AVB</th>\n",
              "      <td>0.900000</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.996997</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.873498</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CLBBB</th>\n",
              "      <td>0.952381</td>\n",
              "      <td>0.829016</td>\n",
              "      <td>0.983903</td>\n",
              "      <td>0.886427</td>\n",
              "      <td>0.906459</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CRBBB</th>\n",
              "      <td>0.788889</td>\n",
              "      <td>0.771739</td>\n",
              "      <td>0.968227</td>\n",
              "      <td>0.780220</td>\n",
              "      <td>0.869983</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PAC</th>\n",
              "      <td>0.666667</td>\n",
              "      <td>0.491803</td>\n",
              "      <td>0.976153</td>\n",
              "      <td>0.566038</td>\n",
              "      <td>0.733978</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>VPC</th>\n",
              "      <td>0.814815</td>\n",
              "      <td>0.758621</td>\n",
              "      <td>0.975124</td>\n",
              "      <td>0.785714</td>\n",
              "      <td>0.866873</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>STD_</th>\n",
              "      <td>0.769231</td>\n",
              "      <td>0.454545</td>\n",
              "      <td>0.995509</td>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.725027</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>STE_</th>\n",
              "      <td>0.844444</td>\n",
              "      <td>0.542857</td>\n",
              "      <td>0.988710</td>\n",
              "      <td>0.660870</td>\n",
              "      <td>0.765783</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      Precision    Recall Specificity  F1 score balanced_accuracy_score\n",
              "                                                                       \n",
              "NORM   0.615385  0.888889    0.935275  0.727273                0.912082\n",
              "AFIB   0.862903  0.869919    0.970018  0.866397                0.919968\n",
              "1AVB   0.900000  0.750000    0.996997  0.818182                0.873498\n",
              "CLBBB  0.952381  0.829016    0.983903  0.886427                0.906459\n",
              "CRBBB  0.788889  0.771739    0.968227  0.780220                0.869983\n",
              "PAC    0.666667  0.491803    0.976153  0.566038                0.733978\n",
              "VPC    0.814815  0.758621    0.975124  0.785714                0.866873\n",
              "STD_   0.769231  0.454545    0.995509  0.571429                0.725027\n",
              "STE_   0.844444  0.542857    0.988710  0.660870                0.765783"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nm56ZNCTmg4p",
        "outputId": "936d32fd-61cb-4bb2-d21a-26abe7c0f843"
      },
      "source": [
        "!pip install catboost"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting catboost\n",
            "  Downloading catboost-0.26-cp37-none-manylinux1_x86_64.whl (69.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 69.2 MB 4.9 kB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from catboost) (1.4.1)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.7/dist-packages (from catboost) (4.4.1)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from catboost) (1.19.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from catboost) (1.15.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from catboost) (3.2.2)\n",
            "Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.7/dist-packages (from catboost) (1.1.5)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.7/dist-packages (from catboost) (0.10.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24.0->catboost) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24.0->catboost) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->catboost) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->catboost) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->catboost) (1.3.1)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.7/dist-packages (from plotly->catboost) (1.3.3)\n",
            "Installing collected packages: catboost\n",
            "Successfully installed catboost-0.26\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 236
        },
        "id": "piiLzp_mXcdu",
        "outputId": "8a327dec-ad14-435c-96d2-de3d03d05eb9"
      },
      "source": [
        "from catboost import CatBoostClassifier\n",
        "\n",
        "cat_estimator = CatBoostClassifier(\n",
        "                # iterations=10, \n",
        "                learning_rate=0.01, depth=10 , n_estimators=100 ,\n",
        "                )\n",
        "\n",
        "multilabel_model = MultiOutputClassifier(cat_estimator)\n",
        "\n",
        "\n",
        "multilabel_model.fit(intermediate_output, y_train,  \n",
        "                      # cat_features=cat_features, \n",
        "                      # eval_set=(intermediate_test_output, y_val),\n",
        "                      # verbose=True\n",
        ")\n",
        "\n",
        "# print('CatBoost model is fitted: ' + str(multilabel_model.is_fitted()))\n",
        "print('CatBoost model parameters:')\n",
        "print(multilabel_model.get_params())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-6bf28fb3c07a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m                 )\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mmultilabel_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMultiOutputClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcat_estimator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'MultiOutputClassifier' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "8nz3SX2qm64_",
        "outputId": "504132f7-a274-4ab6-89cf-4be75193c039"
      },
      "source": [
        "\n",
        "evaluate_experiment(y_val, multilabel_model.predict(intermediate_test_output))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.700877</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0   0.700877"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "xC61naW2m_tO",
        "outputId": "49445aa4-5b24-4f37-a3d3-bed7e037f6f0"
      },
      "source": [
        "evaluate_experiment(y_train, multilabel_model.predict(intermediate_output))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.913026</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0   0.913026"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yYTtPXx-qKZl"
      },
      "source": [
        "#sampling frequency=500\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2GLCgvEiqkTI",
        "outputId": "6125802e-c206-4ac8-9895-efaa19b6a501"
      },
      "source": [
        "from utils import utils\n",
        "\n",
        "sampling_frequency=500\n",
        "datafolder= '/gdrive/My Drive/ICBEB/'\n",
        "task='all'\n",
        "outputfolder='../output0/'\n",
        "\n",
        "# Load data\n",
        "data, raw_labels = utils.load_dataset(datafolder, sampling_frequency)\n",
        "# Preprocess label data\n",
        "labels = utils.compute_label_aggregations(raw_labels, datafolder, task)\n",
        "# Select relevant data and convert to one-hot\n",
        "data, labels, Y, _ = utils.select_data(data, labels, task, min_samples=0, outputfolder=outputfolder)\n",
        "\n",
        "# 1-9 for training \n",
        "X_train = data[labels.strat_fold < 10]\n",
        "y_train = Y[labels.strat_fold < 10]\n",
        "# 10 for validation\n",
        "X_val = data[labels.strat_fold == 10]\n",
        "y_val = Y[labels.strat_fold == 10]\n",
        "\n",
        "num_classes = 9         # <=== number of classes in the finetuning dataset\n",
        "input_shape = [1000,12] # <=== shape of samples, [None, 12] in case of different lengths\n",
        "\n",
        "X_train.shape, y_train.shape, X_val.shape, y_val.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((6187,), (6187, 9), (690,), (690, 9))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EgcQTG7Uh7l_"
      },
      "source": [
        "data.shape\n",
        "data = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_vApeTcWAgU6",
        "outputId": "c64de957-9889-478a-e001-cb24d2cb9fe9"
      },
      "source": [
        "min = X_train[0].shape[0]\n",
        "for i in range(len(X_train)):\n",
        "    if X_train[i].shape[0] < min:\n",
        "        min = X_train[i].shape[0]\n",
        "min\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ez-BiuPGcB4r",
        "outputId": "dfeffb58-c148-403e-b8b6-e510fabcc655"
      },
      "source": [
        "p = X_train[0][:,0]\n",
        "p.shape[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7500"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "49YjHfV0qkTP",
        "outputId": "4bef4fd0-f6f9-41fe-d3eb-67929393e797"
      },
      "source": [
        "X_tr = []\n",
        "for i in range(len(X_train)):\n",
        "    x = []\n",
        "    for j in range(12):\n",
        "        p = X_train[i][:,j]\n",
        "        # print(p.shape[0])\n",
        "        if p.shape[0] < 4000:\n",
        "            d = abs(p.shape[0]-4000)//2\n",
        "            p = np.pad(p,(d,d))\n",
        "            # print(p.shape[0])\n",
        "            if p.shape[0] != 4000:\n",
        "                p = np.pad(p,(1,0))\n",
        "                # print(p.shape)\n",
        "        else:\n",
        "            p = X_train[i][:4000,j]\n",
        "        x.append(p)\n",
        "    X_tr.append(np.transpose(x))\n",
        "X_tr = np.array(X_tr)\n",
        "X_tr.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(6187, 4000, 12)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cKPexZYehCQZ"
      },
      "source": [
        "X_train = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "geMECpoUqkTR",
        "outputId": "0c45cb56-3654-4617-b72f-9d9ddff21ddc"
      },
      "source": [
        "X_te = []\n",
        "for i in range(len(X_val)):\n",
        "    x = []\n",
        "    for j in range(12):\n",
        "        p = X_val[i][:,j]\n",
        "        # print(p.shape[0])\n",
        "        if p.shape[0] < 4000:\n",
        "            d = abs(p.shape[0]-4000)//2\n",
        "            p = np.pad(p,(d,d))\n",
        "            # print(p.shape[0])\n",
        "            if p.shape[0] != 4000:\n",
        "                p = np.pad(p,(1,0))\n",
        "                print(p.shape)\n",
        "        else:\n",
        "            p = X_val[i][:4000,j]\n",
        "        x.append(p)\n",
        "    X_te.append(np.transpose(x))\n",
        "X_te = np.array(X_te)\n",
        "X_te.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(690, 4000, 12)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s4wrD99pn4Q4"
      },
      "source": [
        "X_train = []\n",
        "X_val = []\n",
        "for i in range(len(X_tr)):\n",
        "    x = (X_tr[i] - np.mean(X_tr[i]))/(np.std(X_tr[i])+1e-30)\n",
        "    X_train.append(x)\n",
        "for i in range(len(X_te)):\n",
        "    x = (X_te[i] - np.mean(X_te[i]))/(np.std(X_te[i])+1e-30)\n",
        "    X_val.append(x)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_McuwhWHp-lo"
      },
      "source": [
        "X_tr = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uyh1aMn3qOiZ"
      },
      "source": [
        "X_train = np.array(X_train)\n",
        "X_val = np.array(X_val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YzJF3Ga1qkTT"
      },
      "source": [
        "main_input = Input(shape=(4000,12), dtype='float32', name='main_input')\n",
        "x = Convolution1D(12, 3, padding='same')(main_input)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 24, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 24, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 24, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 24, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 3, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Convolution1D(12, 48, strides = 2, padding='same')(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "cnnout = Dropout(0.2)(x)\n",
        "x = Bidirectional(CuDNNGRU(12, input_shape=(125,12),return_sequences=True,return_state=False))(cnnout)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = AttentionWithContext()(x)\n",
        "x = BatchNormalization()(x)\n",
        "x = LeakyReLU(alpha=0.3)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "main_output = Dense(num_classes,activation='sigmoid')(x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hfJUwUWHqkTV",
        "outputId": "d6211768-4be4-4898-aca9-c4fa009a088b"
      },
      "source": [
        "model = Model(main_input,main_output)\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "main_input (InputLayer)      [(None, 4000, 12)]        0         \n",
            "_________________________________________________________________\n",
            "conv1d (Conv1D)              (None, 4000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu (LeakyReLU)      (None, 4000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_1 (Conv1D)            (None, 4000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_1 (LeakyReLU)    (None, 4000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_2 (Conv1D)            (None, 2000, 12)          3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_2 (LeakyReLU)    (None, 2000, 12)          0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 2000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_3 (Conv1D)            (None, 2000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_3 (LeakyReLU)    (None, 2000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_4 (Conv1D)            (None, 2000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_4 (LeakyReLU)    (None, 2000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_5 (Conv1D)            (None, 1000, 12)          3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_5 (LeakyReLU)    (None, 1000, 12)          0         \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 1000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_6 (Conv1D)            (None, 1000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_6 (LeakyReLU)    (None, 1000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_7 (Conv1D)            (None, 1000, 12)          444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_7 (LeakyReLU)    (None, 1000, 12)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_8 (Conv1D)            (None, 500, 12)           3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_8 (LeakyReLU)    (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_9 (Conv1D)            (None, 500, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_9 (LeakyReLU)    (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_10 (Conv1D)           (None, 500, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_10 (LeakyReLU)   (None, 500, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_11 (Conv1D)           (None, 250, 12)           3468      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_11 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_3 (Dropout)          (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_12 (Conv1D)           (None, 250, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_12 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_13 (Conv1D)           (None, 250, 12)           444       \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_13 (LeakyReLU)   (None, 250, 12)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_14 (Conv1D)           (None, 125, 12)           6924      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_14 (LeakyReLU)   (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 125, 12)           0         \n",
            "_________________________________________________________________\n",
            "bidirectional (Bidirectional (None, 125, 24)           1872      \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_15 (LeakyReLU)   (None, 125, 24)           0         \n",
            "_________________________________________________________________\n",
            "dropout_5 (Dropout)          (None, 125, 24)           0         \n",
            "_________________________________________________________________\n",
            "attention_with_context (Atte (None, 24)                624       \n",
            "_________________________________________________________________\n",
            "batch_normalization (BatchNo (None, 24)                96        \n",
            "_________________________________________________________________\n",
            "leaky_re_lu_16 (LeakyReLU)   (None, 24)                0         \n",
            "_________________________________________________________________\n",
            "dropout_6 (Dropout)          (None, 24)                0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 9)                 225       \n",
            "=================================================================\n",
            "Total params: 28,053\n",
            "Trainable params: 28,005\n",
            "Non-trainable params: 48\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oCHvo1StqkTW"
      },
      "source": [
        "model.compile(optimizer=Adam(learning_rate=1e-4),\n",
        "                loss='binary_crossentropy',\n",
        "                metrics=['AUC'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EUrdEIcFqkTY",
        "outputId": "d8ba597d-31d7-4092-be29-5988dc2c824e"
      },
      "source": [
        "model.fit(X_train, y_train, \n",
        "          validation_split=0.1,\n",
        "          epochs=20,\n",
        "          batch_size=64,\n",
        "          shuffle=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "87/87 [==============================] - 4s 47ms/step - loss: 0.1083 - auc: 0.9762 - val_loss: 0.1266 - val_auc: 0.9645\n",
            "Epoch 2/20\n",
            "87/87 [==============================] - 4s 43ms/step - loss: 0.1093 - auc: 0.9750 - val_loss: 0.1264 - val_auc: 0.9641\n",
            "Epoch 3/20\n",
            "87/87 [==============================] - 4s 46ms/step - loss: 0.1093 - auc: 0.9758 - val_loss: 0.1278 - val_auc: 0.9650\n",
            "Epoch 4/20\n",
            "87/87 [==============================] - 4s 46ms/step - loss: 0.1103 - auc: 0.9751 - val_loss: 0.1258 - val_auc: 0.9644\n",
            "Epoch 5/20\n",
            "87/87 [==============================] - 4s 45ms/step - loss: 0.1075 - auc: 0.9771 - val_loss: 0.1281 - val_auc: 0.9634\n",
            "Epoch 6/20\n",
            "87/87 [==============================] - 4s 45ms/step - loss: 0.1075 - auc: 0.9762 - val_loss: 0.1270 - val_auc: 0.9643\n",
            "Epoch 7/20\n",
            "87/87 [==============================] - 4s 44ms/step - loss: 0.1091 - auc: 0.9748 - val_loss: 0.1264 - val_auc: 0.9648\n",
            "Epoch 8/20\n",
            "87/87 [==============================] - 4s 43ms/step - loss: 0.1087 - auc: 0.9755 - val_loss: 0.1250 - val_auc: 0.9652\n",
            "Epoch 9/20\n",
            "87/87 [==============================] - 4s 45ms/step - loss: 0.1086 - auc: 0.9755 - val_loss: 0.1292 - val_auc: 0.9641\n",
            "Epoch 10/20\n",
            "87/87 [==============================] - 4s 45ms/step - loss: 0.1078 - auc: 0.9760 - val_loss: 0.1282 - val_auc: 0.9648\n",
            "Epoch 11/20\n",
            "87/87 [==============================] - 4s 45ms/step - loss: 0.1072 - auc: 0.9762 - val_loss: 0.1276 - val_auc: 0.9641\n",
            "Epoch 12/20\n",
            "87/87 [==============================] - 4s 44ms/step - loss: 0.1084 - auc: 0.9755 - val_loss: 0.1270 - val_auc: 0.9644\n",
            "Epoch 13/20\n",
            "87/87 [==============================] - 4s 45ms/step - loss: 0.1091 - auc: 0.9756 - val_loss: 0.1276 - val_auc: 0.9643\n",
            "Epoch 14/20\n",
            "87/87 [==============================] - 4s 45ms/step - loss: 0.1057 - auc: 0.9770 - val_loss: 0.1275 - val_auc: 0.9647\n",
            "Epoch 15/20\n",
            "87/87 [==============================] - 4s 45ms/step - loss: 0.1072 - auc: 0.9762 - val_loss: 0.1286 - val_auc: 0.9641\n",
            "Epoch 16/20\n",
            "87/87 [==============================] - 4s 43ms/step - loss: 0.1068 - auc: 0.9763 - val_loss: 0.1273 - val_auc: 0.9648\n",
            "Epoch 17/20\n",
            "87/87 [==============================] - 4s 43ms/step - loss: 0.1083 - auc: 0.9751 - val_loss: 0.1271 - val_auc: 0.9648\n",
            "Epoch 18/20\n",
            "87/87 [==============================] - 4s 44ms/step - loss: 0.1064 - auc: 0.9771 - val_loss: 0.1287 - val_auc: 0.9649\n",
            "Epoch 19/20\n",
            "87/87 [==============================] - 4s 43ms/step - loss: 0.1065 - auc: 0.9765 - val_loss: 0.1254 - val_auc: 0.9648\n",
            "Epoch 20/20\n",
            "87/87 [==============================] - 4s 43ms/step - loss: 0.1068 - auc: 0.9758 - val_loss: 0.1280 - val_auc: 0.9648\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7ff8c4530fd0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "e0Z_tcngrBKT",
        "outputId": "dc67f622-b35e-49f8-9242-a19cc0b933e4"
      },
      "source": [
        "y_pred = model.predict(X_val)\n",
        "utils.evaluate_experiment(y_val, y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.952921</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0   0.952921"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "WhuMetYRqkTa",
        "outputId": "ce805aaf-fead-4870-93a5-1578bc3c6f59"
      },
      "source": [
        "y_pred = model.predict(X_te)\n",
        "utils.evaluate_experiment(y_val, y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.95758</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0    0.95758"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c8jtLG1qqkTb",
        "outputId": "de6fa8bc-78a2-4187-f77b-189caabe83f2"
      },
      "source": [
        "from sklearn.metrics import classification_report, fbeta_score, roc_auc_score, roc_curve, roc_curve, auc\n",
        "\n",
        "def find_optimal_cutoff_threshold(target, predicted):\n",
        "    \"\"\" \n",
        "    Find the optimal probability cutoff point for a classification model related to event rate\n",
        "    \"\"\"\n",
        "    fpr, tpr, threshold = roc_curve(target, predicted)\n",
        "    optimal_idx = np.argmax(tpr - fpr)\n",
        "    optimal_threshold = threshold[optimal_idx]\n",
        "    return optimal_threshold\n",
        "\n",
        "def find_optimal_cutoff_thresholds(y_true, y_pred):\n",
        "\treturn [find_optimal_cutoff_threshold(y_true[:,i], y_pred[:,i]) for i in range(y_true.shape[1])]\n",
        "\n",
        "thresholds =  find_optimal_cutoff_thresholds_for_Gbeta(y_val , y_pred)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/9 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "optimize thresholds with respect to G_beta\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 9/9 [00:07<00:00,  1.14it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fi-NwuxIqkTd",
        "outputId": "404404aa-59b8-4d13-f0cd-4ea520610bd8"
      },
      "source": [
        "y_pred_binary = apply_thresholds(y_pred, thresholds)\n",
        "y_pred_binary[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 0, 0, 0, 0, 0, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "mcXWzV1sqkTe",
        "outputId": "5a9ac3e3-cfee-4ac0-ec7b-56a002e09c31"
      },
      "source": [
        "utils.evaluate_experiment(y_val, y_pred_binary)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>macro_auc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.905897</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   macro_auc\n",
              "0   0.905897"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mru7ltNOqkTe"
      },
      "source": [
        "from sklearn.metrics import (confusion_matrix,\n",
        "                             precision_score, recall_score, f1_score,\n",
        "                             precision_recall_curve, average_precision_score , balanced_accuracy_score)\n",
        "diagnosis = ['NORM', 'AFIB', '1AVB', 'CLBBB', 'CRBBB', 'PAC', 'VPC', 'STD_', 'STE_']\n",
        "def specificity_score(y_true, y_pred):\n",
        "    m = confusion_matrix(y_true, y_pred, labels=[0, 1])\n",
        "    spc = m[0, 0] * 1.0 / (m[0, 0] + m[0, 1])\n",
        "    return spc\n",
        "\n",
        "def get_scores(y_true, y_pred, score_fun):\n",
        "    nclasses = np.shape(y_true)[1]\n",
        "    scores = []\n",
        "    for name, fun in score_fun.items():\n",
        "        scores += [[fun(y_true[:, k], y_pred[:, k]) for k in range(nclasses)]]\n",
        "    return np.array(scores).T\n",
        "\n",
        "\n",
        "score_fun = {'Precision': precision_score,\n",
        "             'Recall': recall_score, 'Specificity': specificity_score,\n",
        "             'F1 score': f1_score , 'balanced accuracy': balanced_accuracy_score}\n",
        "\n",
        "scores_list = []\n",
        "for y_pred in [y_pred_binary]:\n",
        "    # Compute scores\n",
        "    scores = get_scores(y_val, y_pred, score_fun)\n",
        "    # Put them into a data frame\n",
        "    scores_df = pd.DataFrame(scores,index=diagnosis, columns=score_fun.keys())\n",
        "    # Append\n",
        "    scores_list.append(scores_df)\n",
        "# Concatenate dataframes\n",
        "scores_all_df = pd.concat(scores_list, axis=1, keys=[''])\n",
        "# Change multiindex levels\n",
        "scores_all_df = scores_all_df.swaplevel(0, 1, axis=1)\n",
        "scores_all_df = scores_all_df.reindex(level=0, columns=score_fun.keys())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        },
        "id": "cU8NbclpqkTi",
        "outputId": "bf528cf4-5e30-4dbe-8437-af25d1672208"
      },
      "source": [
        "#Gbeta thershold\n",
        "scores_all_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Specificity</th>\n",
              "      <th>F1 score</th>\n",
              "      <th>balanced accuracy</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>NORM</th>\n",
              "      <td>0.873418</td>\n",
              "      <td>0.958333</td>\n",
              "      <td>0.983819</td>\n",
              "      <td>0.913907</td>\n",
              "      <td>0.971076</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AFIB</th>\n",
              "      <td>0.879699</td>\n",
              "      <td>0.951220</td>\n",
              "      <td>0.971781</td>\n",
              "      <td>0.914062</td>\n",
              "      <td>0.961500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1AVB</th>\n",
              "      <td>0.846154</td>\n",
              "      <td>0.916667</td>\n",
              "      <td>0.993994</td>\n",
              "      <td>0.880000</td>\n",
              "      <td>0.955330</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CLBBB</th>\n",
              "      <td>0.896714</td>\n",
              "      <td>0.989637</td>\n",
              "      <td>0.955734</td>\n",
              "      <td>0.940887</td>\n",
              "      <td>0.972686</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CRBBB</th>\n",
              "      <td>0.783505</td>\n",
              "      <td>0.826087</td>\n",
              "      <td>0.964883</td>\n",
              "      <td>0.804233</td>\n",
              "      <td>0.895485</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PAC</th>\n",
              "      <td>0.436893</td>\n",
              "      <td>0.737705</td>\n",
              "      <td>0.907790</td>\n",
              "      <td>0.548780</td>\n",
              "      <td>0.822748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>VPC</th>\n",
              "      <td>0.740000</td>\n",
              "      <td>0.850575</td>\n",
              "      <td>0.956882</td>\n",
              "      <td>0.791444</td>\n",
              "      <td>0.903728</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>STD_</th>\n",
              "      <td>0.652174</td>\n",
              "      <td>0.681818</td>\n",
              "      <td>0.988024</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>0.834921</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>STE_</th>\n",
              "      <td>0.842105</td>\n",
              "      <td>0.685714</td>\n",
              "      <td>0.985484</td>\n",
              "      <td>0.755906</td>\n",
              "      <td>0.835599</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      Precision    Recall Specificity  F1 score balanced accuracy\n",
              "                                                                 \n",
              "NORM   0.873418  0.958333    0.983819  0.913907          0.971076\n",
              "AFIB   0.879699  0.951220    0.971781  0.914062          0.961500\n",
              "1AVB   0.846154  0.916667    0.993994  0.880000          0.955330\n",
              "CLBBB  0.896714  0.989637    0.955734  0.940887          0.972686\n",
              "CRBBB  0.783505  0.826087    0.964883  0.804233          0.895485\n",
              "PAC    0.436893  0.737705    0.907790  0.548780          0.822748\n",
              "VPC    0.740000  0.850575    0.956882  0.791444          0.903728\n",
              "STD_   0.652174  0.681818    0.988024  0.666667          0.834921\n",
              "STE_   0.842105  0.685714    0.985484  0.755906          0.835599"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 644
        },
        "id": "JXs7Z6BFqkTj",
        "outputId": "17e69482-ff87-4150-db6d-f036da418797"
      },
      "source": [
        "# %% Confusion matrices (Supplementary Table 1)\n",
        "from sklearn.metrics import (confusion_matrix,\n",
        "                             precision_score, recall_score, f1_score,\n",
        "                             precision_recall_curve, average_precision_score)\n",
        "\n",
        "import xarray as xr\n",
        "\n",
        "M = [[confusion_matrix(y_val[:, k], y_pred[:, k], labels=[0, 1])\n",
        "      for k in range(9)] for y_pred in [y_pred_binary]]\n",
        "\n",
        "M_xarray = xr.DataArray(np.array(M),\n",
        "                        dims=['predictor', 'diagnosis', 'true label', 'predicted label'],\n",
        "                        coords={'predictor': ['CNN+RNN'],\n",
        "                                'diagnosis': diagnosis,\n",
        "                                'true label': ['not present', 'present'],\n",
        "                                'predicted label': ['not present', 'present']})\n",
        "confusion_matrices = M_xarray.to_dataframe('n')\n",
        "confusion_matrices = confusion_matrices.reorder_levels([1, 2, 3, 0], axis=0)\n",
        "confusion_matrices = confusion_matrices.unstack()\n",
        "confusion_matrices = confusion_matrices.unstack()\n",
        "confusion_matrices = confusion_matrices['n']\n",
        "confusion_matrices"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>predictor</th>\n",
              "      <th colspan=\"2\" halign=\"left\">CNN+RNN</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>predicted label</th>\n",
              "      <th>not present</th>\n",
              "      <th>present</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>diagnosis</th>\n",
              "      <th>true label</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">NORM</th>\n",
              "      <th>not present</th>\n",
              "      <td>608</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>3</td>\n",
              "      <td>69</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">AFIB</th>\n",
              "      <th>not present</th>\n",
              "      <td>551</td>\n",
              "      <td>16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>6</td>\n",
              "      <td>117</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1AVB</th>\n",
              "      <th>not present</th>\n",
              "      <td>662</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>2</td>\n",
              "      <td>22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">CLBBB</th>\n",
              "      <th>not present</th>\n",
              "      <td>475</td>\n",
              "      <td>22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>2</td>\n",
              "      <td>191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">CRBBB</th>\n",
              "      <th>not present</th>\n",
              "      <td>577</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>16</td>\n",
              "      <td>76</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">PAC</th>\n",
              "      <th>not present</th>\n",
              "      <td>571</td>\n",
              "      <td>58</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>16</td>\n",
              "      <td>45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">VPC</th>\n",
              "      <th>not present</th>\n",
              "      <td>577</td>\n",
              "      <td>26</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>13</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">STD_</th>\n",
              "      <th>not present</th>\n",
              "      <td>660</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>7</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">STE_</th>\n",
              "      <th>not present</th>\n",
              "      <td>611</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>present</th>\n",
              "      <td>22</td>\n",
              "      <td>48</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "predictor                 CNN+RNN        \n",
              "predicted label       not present present\n",
              "diagnosis true label                     \n",
              "NORM      not present         608      10\n",
              "          present               3      69\n",
              "AFIB      not present         551      16\n",
              "          present               6     117\n",
              "1AVB      not present         662       4\n",
              "          present               2      22\n",
              "CLBBB     not present         475      22\n",
              "          present               2     191\n",
              "CRBBB     not present         577      21\n",
              "          present              16      76\n",
              "PAC       not present         571      58\n",
              "          present              16      45\n",
              "VPC       not present         577      26\n",
              "          present              13      74\n",
              "STD_      not present         660       8\n",
              "          present               7      15\n",
              "STE_      not present         611       9\n",
              "          present              22      48"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eHMvgjVClRjx"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}