GraphClass.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import stellargraph as sg\n",
    "from stellargraph.mapper import PaddedGraphGenerator\n",
    "from stellargraph.layer import GCNSupervisedGraphClassification\n",
    "from stellargraph import StellarGraph\n",
    "\n",
    "from stellargraph import datasets\n",
    "\n",
    "from sklearn import model_selection\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "from tensorflow.keras import Model\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.losses import binary_crossentropy\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.2.1'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sg.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mydf = pd.read_csv(\"/Users/morganoneka/Box/My Stuff/GraphClassification/GiottoOutput/Chronic Pancreatitis/318_1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "mydf['weight'] = [abs(x) for x in mydf['weight']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>weight</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>T-reg</td>\n",
       "      <td>Tumor</td>\n",
       "      <td>2.424922</td>\n",
       "      <td>Depletion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Other</td>\n",
       "      <td>Tumor</td>\n",
       "      <td>1.869055</td>\n",
       "      <td>Depletion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Helper T-Cell</td>\n",
       "      <td>Tumor</td>\n",
       "      <td>1.469400</td>\n",
       "      <td>Depletion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>APC</td>\n",
       "      <td>Tumor</td>\n",
       "      <td>1.286881</td>\n",
       "      <td>Depletion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CTL</td>\n",
       "      <td>Other</td>\n",
       "      <td>1.051024</td>\n",
       "      <td>Depletion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Other</td>\n",
       "      <td>T-reg</td>\n",
       "      <td>0.783806</td>\n",
       "      <td>Depletion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CTL</td>\n",
       "      <td>Tumor</td>\n",
       "      <td>0.535342</td>\n",
       "      <td>Depletion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>APC</td>\n",
       "      <td>APC</td>\n",
       "      <td>2.403178</td>\n",
       "      <td>Enrichment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Helper T-Cell</td>\n",
       "      <td>Helper T-Cell</td>\n",
       "      <td>2.217591</td>\n",
       "      <td>Enrichment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>APC</td>\n",
       "      <td>Helper T-Cell</td>\n",
       "      <td>1.538010</td>\n",
       "      <td>Enrichment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>1.340963</td>\n",
       "      <td>Enrichment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Tumor</td>\n",
       "      <td>Tumor</td>\n",
       "      <td>0.525335</td>\n",
       "      <td>Enrichment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>APC</td>\n",
       "      <td>Other</td>\n",
       "      <td>0.278631</td>\n",
       "      <td>Enrichment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Helper T-Cell</td>\n",
       "      <td>Other</td>\n",
       "      <td>0.240492</td>\n",
       "      <td>Enrichment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           source         target    weight        type\n",
       "0           T-reg          Tumor  2.424922   Depletion\n",
       "1           Other          Tumor  1.869055   Depletion\n",
       "2   Helper T-Cell          Tumor  1.469400   Depletion\n",
       "3             APC          Tumor  1.286881   Depletion\n",
       "4             CTL          Other  1.051024   Depletion\n",
       "5           Other          T-reg  0.783806   Depletion\n",
       "6             CTL          Tumor  0.535342   Depletion\n",
       "7             APC            APC  2.403178  Enrichment\n",
       "8   Helper T-Cell  Helper T-Cell  2.217591  Enrichment\n",
       "9             APC  Helper T-Cell  1.538010  Enrichment\n",
       "10          Other          Other  1.340963  Enrichment\n",
       "11          Tumor          Tumor  0.525335  Enrichment\n",
       "12            APC          Other  0.278631  Enrichment\n",
       "13  Helper T-Cell          Other  0.240492  Enrichment"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mydf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "weighted_features = StellarGraph(edges=mydf, edge_type_column='type')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StellarGraph: Undirected multigraph\n",
      " Nodes: 6, Edges: 14\n",
      "\n",
      " Node types:\n",
      "  default: [6]\n",
      "    Features: none\n",
      "    Edge types: default-Depletion->default, default-Enrichment->default\n",
      "\n",
      " Edge types:\n",
      "    default-Enrichment->default: [7]\n",
      "        Weights: range=[0.240492, 2.40318], mean=1.2206, std=0.898056\n",
      "        Features: none\n",
      "    default-Depletion->default: [7]\n",
      "        Weights: range=[0.535342, 2.42492], mean=1.34578, std=0.647639\n",
      "        Features: none\n"
     ]
    }
   ],
   "source": [
    "print(weighted_features.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_dirs = [\"/Users/morganoneka/Box/My Stuff/GraphClassification/GiottoOutput/\" + x for x in ['Chronic Pancreatitis', 'IPMN', 'MCN', 'PanIN', 'PDAC']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/morganoneka/Box/My Stuff/GraphClassification/GiottoOutput/Chronic Pancreatitis',\n",
       " '/Users/morganoneka/Box/My Stuff/GraphClassification/GiottoOutput/IPMN',\n",
       " '/Users/morganoneka/Box/My Stuff/GraphClassification/GiottoOutput/MCN',\n",
       " '/Users/morganoneka/Box/My Stuff/GraphClassification/GiottoOutput/PanIN',\n",
       " '/Users/morganoneka/Box/My Stuff/GraphClassification/GiottoOutput/PDAC']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_dirs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = [len(os.listdir(x)) for x in file_dirs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[110, 176, 34, 82, 230]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_graphs(dirname):\n",
    "    my_graphs=[]\n",
    "    files = os.listdir(dirname)\n",
    "    csvs = [x for x in files if x.endswith(\".csv\")]\n",
    "    for f in csvs:\n",
    "        mydf = pd.read_csv(dirname + \"/\" + f)\n",
    "        mydf['weight'] = [abs(x) for x in mydf['weight']]\n",
    "        # 1 = tumor\n",
    "        # 2 = immune \n",
    "        # 3 = other\n",
    "#         node_data = pd.DataFrame(\n",
    "#             {\"celltype\": [\"1\", \"2\", \"2\", \"2\", \"2\", \"3\"]}, index=[\"Tumor\", \"APC\", \"Helper T-Cell\", \"CTL\", \"T-reg\", \"Other\"]\n",
    "#         )\n",
    "        node_data = pd.DataFrame(\n",
    "            {\"celltype\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\"]}, index=[\"Tumor\", \"APC\", \"Helper T-Cell\", \"CTL\", \"T-reg\", \"Other\"]\n",
    "        )\n",
    "        weighted_features = StellarGraph(nodes=node_data, edges=mydf, edge_type_column='type')\n",
    "        my_graphs.append(weighted_features)\n",
    "    return(my_graphs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp_graphs = get_graphs(file_dirs[0])\n",
    "ipmn_graphs = get_graphs(file_dirs[1])\n",
    "mcn_graphs = get_graphs(file_dirs[2])\n",
    "panin_graphs = get_graphs(file_dirs[3])\n",
    "pdac_graphs = get_graphs(file_dirs[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def remove_problematic_graphs(grp):\n",
    "#     return ([g for g in grp if g.number_of_edges() > 0])\n",
    "\n",
    "# print(len(cp_graphs))\n",
    "# print(len(remove_problematic_graphs(cp_graphs)))\n",
    "\n",
    "# print(len(ipmn_graphs))\n",
    "# print(len(remove_problematic_graphs(ipmn_graphs)))\n",
    "\n",
    "# print(len(mcn_graphs))\n",
    "# print(len(remove_problematic_graphs(mcn_graphs)))\n",
    "\n",
    "# print(len(panin_graphs))\n",
    "# print(len(remove_problematic_graphs(panin_graphs)))\n",
    "\n",
    "# print(len(pdac_graphs))\n",
    "# print(len(remove_problematic_graphs(pdac_graphs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_graphs = cp_graphs + ipmn_graphs + mcn_graphs + panin_graphs + pdac_graphs\n",
    "# all_graphs = cp_graphs + ipmn_graphs  + panin_graphs + pdac_graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'StellarGraph: Undirected multigraph\\n Nodes: 6, Edges: 13\\n\\n Node types:\\n  default: [6]\\n    Features: float32 vector, length 1\\n    Edge types: default-Depletion->default, default-Enrichment->default\\n\\n Edge types:\\n    default-Enrichment->default: [7]\\n        Weights: range=[0.475616, 1.93021], mean=1.17158, std=0.644993\\n        Features: none\\n    default-Depletion->default: [6]\\n        Weights: range=[0.136256, 4.2493], mean=2.49698, std=1.54048\\n        Features: none'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_graphs[0].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels = [\"CP\" for x in range(idx[0])] + [\"IPMN\" for x in range(idx[1])] + [\"MCN\" for x in range(idx[2])] + [\"PanIN\" for x in range(idx[3])] + [\"PDAC\" for x in range(idx[4])]\n",
    "# labels = [0 for x in range(len(cp_graphs))] + [1 for x in range(len(ipmn_graphs))] + [2 for x in range(len(mcn_graphs))] + [3 for x in range(len(panin_graphs))] + [4 for x in range(len(pdac_graphs))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CP = non-cancer\n",
    "# IPMN = pre-cancer\n",
    "# MCN = pre-cancer\n",
    "# PanIN = pre-cancer\n",
    "# PDAC = cancer\n",
    "labels = [0 for x in range(len(cp_graphs))] + [1 for x in range(len(ipmn_graphs))] + [1 for x in range(len(mcn_graphs))] + [1 for x in range(len(panin_graphs))] + [2 for x in range(len(pdac_graphs))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0      0\n",
      "1      0\n",
      "2      0\n",
      "3      0\n",
      "4      0\n",
      "      ..\n",
      "311    2\n",
      "312    2\n",
      "313    2\n",
      "314    2\n",
      "315    2\n",
      "Length: 316, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "graph_labels = pd.Series(labels)\n",
    "print(graph_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "316"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_graphs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator = PaddedGraphGenerator(graphs=all_graphs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_graph_classification_model(generator):\n",
    "    gc_model = GCNSupervisedGraphClassification(\n",
    "        layer_sizes=[64, 64],\n",
    "        activations=[\"relu\", \"relu\"],\n",
    "        generator=generator,\n",
    "        dropout=0.5,\n",
    "    )\n",
    "    x_inp, x_out = gc_model.in_out_tensors()\n",
    "    predictions = Dense(units=32, activation=\"relu\")(x_out)\n",
    "    predictions = Dense(units=16, activation=\"relu\")(predictions)\n",
    "    predictions = Dense(units=1, activation=\"sigmoid\")(predictions)\n",
    "\n",
    "    # Let's create the Keras model and prepare it for training\n",
    "    model = Model(inputs=x_inp, outputs=predictions)\n",
    "    model.compile(optimizer=Adam(0.005), loss=binary_crossentropy, metrics=[\"acc\"])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 200  # maximum number of training epochs\n",
    "folds = 10  # the number of folds for k-fold cross validation (default was 10)\n",
    "n_repeats = 5  # the number of repeats for repeated k-fold cross validation (default was 5)\n",
    "es = EarlyStopping(\n",
    "    monitor=\"val_loss\", min_delta=0, patience=25, restore_best_weights=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_fold(model, train_gen, test_gen, es, epochs):\n",
    "    history = model.fit(\n",
    "        train_gen, epochs=epochs, validation_data=test_gen, verbose=0, callbacks=[es],\n",
    "    )\n",
    "    # calculate performance on the test data and return along with history\n",
    "    test_metrics = model.evaluate(test_gen, verbose=0)\n",
    "    test_acc = test_metrics[model.metrics_names.index(\"acc\")]\n",
    "\n",
    "    return history, test_acc\n",
    "\n",
    "def get_generators(train_index, test_index, graph_labels, batch_size):\n",
    "    train_gen = generator.flow(\n",
    "        train_index, targets=graph_labels.iloc[train_index].values, batch_size=batch_size\n",
    "    )\n",
    "    test_gen = generator.flow(\n",
    "        test_index, targets=graph_labels.iloc[test_index].values, batch_size=batch_size\n",
    "    )\n",
    "\n",
    "    return train_gen, test_gen\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training and evaluating on fold 1 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 2 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 3 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 4 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 5 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 6 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 7 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 8 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 9 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 10 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 11 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 12 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 13 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 14 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 15 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 16 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 17 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 18 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 19 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 20 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 21 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 22 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 23 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 24 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 25 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 26 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 27 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 28 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 29 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 30 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 31 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 32 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 33 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 34 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 35 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 36 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 37 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 38 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 39 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 40 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 41 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 42 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 43 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 44 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 45 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 46 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 47 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 48 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 49 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Training and evaluating on fold 50 out of 50...\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n"
     ]
    }
   ],
   "source": [
    "test_accs = []\n",
    "\n",
    "stratified_folds = model_selection.RepeatedStratifiedKFold(\n",
    "    n_splits=folds, n_repeats=n_repeats\n",
    ").split(graph_labels, graph_labels)\n",
    "\n",
    "for i, (train_index, test_index) in enumerate(stratified_folds):\n",
    "    print(f\"Training and evaluating on fold {i+1} out of {folds * n_repeats}...\")\n",
    "    train_gen, test_gen = get_generators(\n",
    "        train_index, test_index, graph_labels, batch_size=30\n",
    "    )\n",
    "\n",
    "    model = create_graph_classification_model(generator)\n",
    "\n",
    "    history, acc = train_fold(model, train_gen, test_gen, es, epochs)\n",
    "\n",
    "    test_accs.append(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.46875,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.46875,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.46875,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.46875,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.46875,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.46875,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.46875,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.46875,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.46875,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.4375,\n",
       " 0.46875,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095,\n",
       " 0.48387095]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_accs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}