sample-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tHF7K8ptOKyd"
   },
   "source": [
    "# Sample Cloud AI Platform Notebook: Predicting Visitor Behaviour\n",
    "Predict if a visitor will add items to the cart using their browsing session data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GhPPnhS_OKyj"
   },
   "source": [
    "# Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "RJqWJZNZOKyk"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "from datetime import datetime\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import feature_column\n",
    "import tensorflow_io as tfio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uX1d-Oh3OKyl"
   },
   "source": [
    "# Load and plot some data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "b13tNL4xOKyl"
   },
   "source": [
    "Let's use the Google Cloud datawharehouse [BigQuery](https://cloud.google.com/bigquery/docs/introduction) to query the data.  \n",
    "The BigQuery client library provides a cell magic ```%%bigquery``` which runs a SQL query and returns the results as a Pandas DataFrame.  \n",
    "Use the cell magic to query a sample of data and save the results in the ```train_df``` DataFrame:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "pwiqKSb_OKyl"
   },
   "outputs": [],
   "source": [
    "%%bigquery train_df\n",
    "SELECT \n",
    "*\n",
    "FROM\n",
    "  `challenge.training_data`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "r_fniemoOKym"
   },
   "source": [
    "Show the first few rows of the DataFrame:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "2brTRhBqOKym",
    "outputId": "f92bdce0-1b04-4f16-97f6-69cce8bc01d6"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>JOBID</th>\n",
       "      <th>DAY</th>\n",
       "      <th>HOUR</th>\n",
       "      <th>VERSION</th>\n",
       "      <th>PERFORMANCE</th>\n",
       "      <th>PRECISION</th>\n",
       "      <th>MPLINK</th>\n",
       "      <th>NTNU</th>\n",
       "      <th>MPLINK_NTNU</th>\n",
       "      <th>MBS</th>\n",
       "      <th>...</th>\n",
       "      <th>NBNODES</th>\n",
       "      <th>NBELEM1D</th>\n",
       "      <th>NBELEM2D</th>\n",
       "      <th>NBELEM3D</th>\n",
       "      <th>CLUSTER</th>\n",
       "      <th>NBSERVERS</th>\n",
       "      <th>NBCORE</th>\n",
       "      <th>DATACHECK_TIME</th>\n",
       "      <th>ELAPSEDTIME</th>\n",
       "      <th>TZC_FINAL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>972300</td>\n",
       "      <td>03/03/2020</td>\n",
       "      <td>17:50:30</td>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>POLE</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>...</td>\n",
       "      <td>4507694</td>\n",
       "      <td>14570</td>\n",
       "      <td>4037272</td>\n",
       "      <td>1570802</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>288</td>\n",
       "      <td>446.00</td>\n",
       "      <td>15000</td>\n",
       "      <td>0.0114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>975299</td>\n",
       "      <td>03/04/2020</td>\n",
       "      <td>09:38:09</td>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>POLE</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>...</td>\n",
       "      <td>4507694</td>\n",
       "      <td>14570</td>\n",
       "      <td>4037272</td>\n",
       "      <td>1570802</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>288</td>\n",
       "      <td>446.00</td>\n",
       "      <td>15600</td>\n",
       "      <td>0.0114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>993587</td>\n",
       "      <td>03/06/2020</td>\n",
       "      <td>10:53:17</td>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>POLE</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>...</td>\n",
       "      <td>4507694</td>\n",
       "      <td>14570</td>\n",
       "      <td>4037272</td>\n",
       "      <td>1570802</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>288</td>\n",
       "      <td>446.00</td>\n",
       "      <td>14600</td>\n",
       "      <td>0.0115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>974258</td>\n",
       "      <td>03/04/2020</td>\n",
       "      <td>07:43:46</td>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>PEDESTRIAN</td>\n",
       "      <td>1</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>...</td>\n",
       "      <td>1647532</td>\n",
       "      <td>3760</td>\n",
       "      <td>1673587</td>\n",
       "      <td>103822</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36</td>\n",
       "      <td>74.70</td>\n",
       "      <td>4510</td>\n",
       "      <td>0.0307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891215</td>\n",
       "      <td>01/11/2019</td>\n",
       "      <td>12:48:09</td>\n",
       "      <td>2016.05</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>...</td>\n",
       "      <td>174111</td>\n",
       "      <td>0</td>\n",
       "      <td>173261</td>\n",
       "      <td>0</td>\n",
       "      <td>HPC1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>48</td>\n",
       "      <td>4.42</td>\n",
       "      <td>4060</td>\n",
       "      <td>0.0234</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    JOBID         DAY      HOUR   VERSION PERFORMANCE  PRECISION MPLINK NTNU  \\\n",
       "0  972300  03/03/2020  17:50:30  2018.0.1        POLE          1    YES   NO   \n",
       "1  975299  03/04/2020  09:38:09  2018.0.1        POLE          1    YES   NO   \n",
       "2  993587  03/06/2020  10:53:17  2018.0.1        POLE          1    YES   NO   \n",
       "3  974258  03/04/2020  07:43:46  2018.0.1  PEDESTRIAN          1     NO   NO   \n",
       "4  891215  01/11/2019  12:48:09   2016.05     UNKNOWN          1    YES   NO   \n",
       "\n",
       "  MPLINK_NTNU MBS  ...  NBNODES  NBELEM1D  NBELEM2D  NBELEM3D  CLUSTER  \\\n",
       "0          NO  NO  ...  4507694     14570   4037272   1570802     HPC3   \n",
       "1          NO  NO  ...  4507694     14570   4037272   1570802     HPC3   \n",
       "2          NO  NO  ...  4507694     14570   4037272   1570802     HPC3   \n",
       "3          NO  NO  ...  1647532      3760   1673587    103822     HPC3   \n",
       "4          NO  NO  ...   174111         0    173261         0     HPC1   \n",
       "\n",
       "   NBSERVERS NBCORE  DATACHECK_TIME  ELAPSEDTIME  TZC_FINAL  \n",
       "0        8.0    288          446.00        15000     0.0114  \n",
       "1        8.0    288          446.00        15600     0.0114  \n",
       "2        8.0    288          446.00        14600     0.0115  \n",
       "3        1.0     36           74.70         4510     0.0307  \n",
       "4        2.0     48            4.42         4060     0.0234  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MeS2lkC6OKyn"
   },
   "source": [
    "Show the DataFrame details:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "o9prqHwmOKyo",
    "outputId": "89c79017-53ee-48b4-d657-ea20d15260f6"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VERSION</th>\n",
       "      <th>PERFORMANCE</th>\n",
       "      <th>PRECISION</th>\n",
       "      <th>MPLINK</th>\n",
       "      <th>NTNU</th>\n",
       "      <th>MPLINK_NTNU</th>\n",
       "      <th>MBS</th>\n",
       "      <th>RUNEND</th>\n",
       "      <th>TIMESTEP</th>\n",
       "      <th>NBNODES</th>\n",
       "      <th>NBELEM1D</th>\n",
       "      <th>NBELEM2D</th>\n",
       "      <th>NBELEM3D</th>\n",
       "      <th>CLUSTER</th>\n",
       "      <th>NBSERVERS</th>\n",
       "      <th>NBCORE</th>\n",
       "      <th>DATACHECK_TIME</th>\n",
       "      <th>ELAPSEDTIME</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>POLE</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>110.01</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>4507694</td>\n",
       "      <td>14570</td>\n",
       "      <td>4037272</td>\n",
       "      <td>1570802</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>288</td>\n",
       "      <td>446.00</td>\n",
       "      <td>15000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>POLE</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>110.01</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>4507694</td>\n",
       "      <td>14570</td>\n",
       "      <td>4037272</td>\n",
       "      <td>1570802</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>288</td>\n",
       "      <td>446.00</td>\n",
       "      <td>15600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>POLE</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>110.01</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>4507694</td>\n",
       "      <td>14570</td>\n",
       "      <td>4037272</td>\n",
       "      <td>1570802</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>288</td>\n",
       "      <td>446.00</td>\n",
       "      <td>14600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>PEDESTRIAN</td>\n",
       "      <td>1</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>40.68</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>1647532</td>\n",
       "      <td>3760</td>\n",
       "      <td>1673587</td>\n",
       "      <td>103822</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36</td>\n",
       "      <td>74.70</td>\n",
       "      <td>4510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016.05</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>50.00</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>174111</td>\n",
       "      <td>0</td>\n",
       "      <td>173261</td>\n",
       "      <td>0</td>\n",
       "      <td>HPC1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>48</td>\n",
       "      <td>4.42</td>\n",
       "      <td>4060</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    VERSION PERFORMANCE  PRECISION MPLINK NTNU MPLINK_NTNU MBS  RUNEND  \\\n",
       "0  2018.0.1        POLE          1    YES   NO          NO  NO  110.01   \n",
       "1  2018.0.1        POLE          1    YES   NO          NO  NO  110.01   \n",
       "2  2018.0.1        POLE          1    YES   NO          NO  NO  110.01   \n",
       "3  2018.0.1  PEDESTRIAN          1     NO   NO          NO  NO   40.68   \n",
       "4   2016.05     UNKNOWN          1    YES   NO          NO  NO   50.00   \n",
       "\n",
       "   TIMESTEP  NBNODES  NBELEM1D  NBELEM2D  NBELEM3D CLUSTER  NBSERVERS  NBCORE  \\\n",
       "0   0.00050  4507694     14570   4037272   1570802    HPC3        8.0     288   \n",
       "1   0.00050  4507694     14570   4037272   1570802    HPC3        8.0     288   \n",
       "2   0.00050  4507694     14570   4037272   1570802    HPC3        8.0     288   \n",
       "3   0.00050  1647532      3760   1673587    103822    HPC3        1.0      36   \n",
       "4   0.00005   174111         0    173261         0    HPC1        2.0      48   \n",
       "\n",
       "   DATACHECK_TIME  ELAPSEDTIME  \n",
       "0          446.00        15000  \n",
       "1          446.00        15600  \n",
       "2          446.00        14600  \n",
       "3           74.70         4510  \n",
       "4            4.42         4060  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = train_df.drop([\"JOBID\",\"DAY\",\"HOUR\",\"TZC_FINAL\"], axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "UE_z1L3DOKyo",
    "outputId": "11194f8d-971a-4de3-e7f3-01015e034fde"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['VERSION', 'PERFORMANCE', 'PRECISION', 'MPLINK', 'NTNU', 'MPLINK_NTNU',\n",
      "       'MBS', 'RUNEND', 'TIMESTEP', 'NBNODES', 'NBELEM1D', 'NBELEM2D',\n",
      "       'NBELEM3D', 'CLUSTER', 'NBSERVERS', 'NBCORE', 'DATACHECK_TIME',\n",
      "       'ELAPSEDTIME'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "APyZNoQFOKyo"
   },
   "source": [
    "printing catagorical and numerical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "tLfqilQ2OKyp"
   },
   "outputs": [],
   "source": [
    "numerical = list(df.describe().columns)\n",
    "categorical = [col for col in df.columns if col not in numerical]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "YGGEjALFOKyp",
    "outputId": "eba04943-acc3-4601-ed9b-90fa957a1a03"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['VERSION', 'PERFORMANCE', 'MPLINK', 'NTNU', 'MPLINK_NTNU', 'MBS', 'CLUSTER']\n",
      "['PRECISION', 'RUNEND', 'TIMESTEP', 'NBNODES', 'NBELEM1D', 'NBELEM2D', 'NBELEM3D', 'NBSERVERS', 'NBCORE', 'DATACHECK_TIME', 'ELAPSEDTIME']\n"
     ]
    }
   ],
   "source": [
    "print(categorical)\n",
    "print(numerical)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KGj0BLN2Op86"
   },
   "source": [
    "Show unique values for each catagorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "executionInfo": {
     "elapsed": 520,
     "status": "error",
     "timestamp": 1612176806937,
     "user": {
      "displayName": "Aurélien Galicher",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgqTtOOiEo05zGER22O3SrHzEVPVJ04tJDKf6kP=s64",
      "userId": "09808616516538419425"
     },
     "user_tz": -60
    },
    "id": "0wWbDJygOKyp",
    "outputId": "744ddd60-166d-4a7d-e05f-58ca9e8d179b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'VERSION': ['2018.0.1', '2016.05', '2016.06', '2019.0.2', '2012.7', '2019.0', '2018.0', '2016.01', '2019.0.1', '2019']}\n",
      "{'PERFORMANCE': ['POLE', 'PEDESTRIAN', 'UNKNOWN', 'ECE', 'FRONT', 'RCAR', 'COCKPIT', 'OVERSLAM', 'REAR', 'SIDE', 'WHEEL']}\n",
      "{'MPLINK': ['YES', 'NO']}\n",
      "{'NTNU': ['NO', 'YES']}\n",
      "{'MPLINK_NTNU': ['NO', 'YES']}\n",
      "{'MBS': ['NO', 'USED']}\n",
      "{'CLUSTER': ['HPC3', 'HPC1', 'HPC2']}\n"
     ]
    }
   ],
   "source": [
    "df[categorical].describe()\n",
    "for x in categorical:\n",
    "    print ({x: list(df[x].unique())})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ynh_GH2bOKyq"
   },
   "source": [
    "# Local training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Tqkd3mgUOKyq"
   },
   "source": [
    "First, let's try to train the model locally (from within the notebook)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fv2uyGuZOKyq"
   },
   "source": [
    "## Create a Google Cloud Storage bucket"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rD_cEQobOKyr"
   },
   "source": [
    "We need a way to centrally store and share data across services.  \n",
    "Let's use [Google Cloud Storage](https://cloud.google.com/storage) which is the blob storage service from Google Cloud."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vkP8zKu-OKyr"
   },
   "source": [
    "```gsutil``` is a command line tool for Google Cloud Storage.  \n",
    "In Google Cloud Storage, URI are in the format ```gs://bucket/folder/file``` .  \n",
    "Use ```gsutil mb gs://YourBucketName``` to create a Google Cloud Storage bucket.\n",
    "\n",
    "Your bucket name must be **globally** unique and must contain only lowercase letters, numbers, dashes, underscores, and dots.  \n",
    "**!!! Change the bucket name below with your own!!!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "efaxbwq5OKyr",
    "outputId": "36503f3a-a0fe-480a-a03d-8d1a8a5ff111"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating gs://ml-competition-youripn/...\n",
      "ServiceException: 409 Bucket ml-competition-youripn already exists.\n"
     ]
    }
   ],
   "source": [
    "# Prepare Google Cloud Storage directory to save logs and model\n",
    "BUCKET_NAME = 'ml-competition-youripn' # Create your own unique bucket name\n",
    "!gsutil mb -l EU gs://{BUCKET_NAME}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6yr_YrxZOKys"
   },
   "source": [
    "Use ```gsutil ls gs://YourBucketName``` to list your bucket and make sure it has been correctly created.  \n",
    "Expect no output if the bucket is correctly created."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "90MX2IsQOKys",
    "outputId": "94643123-8a99-4138-cd98-ebc9e47c4ff5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://ml-competition-youripn/latest_model/\n",
      "gs://ml-competition-youripn/training_job_2021_02_03_165524/\n"
     ]
    }
   ],
   "source": [
    "!gsutil ls gs://{BUCKET_NAME}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4gmMzOGYOKys"
   },
   "source": [
    "## Create your trainer package structure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sICzp_Z4OKyt"
   },
   "source": [
    "Now let's create your trainer package structure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "FHCYtF7FOKyt",
    "outputId": "9f731352-e828-40ee-d0a8-cf1830ec3dd2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘./trainer’: File exists\n"
     ]
    }
   ],
   "source": [
    "# Create the Trainer package structure\n",
    "!mkdir ./trainer\n",
    "!touch ./trainer/__init__.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mz00P28uOKyv"
   },
   "source": [
    "**!!! Change the bucket name below with your own!!!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "lA2utUPUOKyv",
    "outputId": "f8ae3df0-f598-400c-9a3c-98a441f25365"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./trainer/task.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./trainer/task.py\n",
    "\n",
    "# Imports\n",
    "import os\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from google.cloud import bigquery\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Declare constants\n",
    "BUCKET_NAME = 'ml-competition-youripn'  # change to YOUR bucket name\n",
    "JOB_DIR = f'gs://{BUCKET_NAME}/latest_model' \n",
    "CATEGORICAL_TYPES = {'VERSION': pd.api.types.CategoricalDtype(['2018.0.1', '2016.05', '2016.06', '2019.0.2', '2012.7', '2019.0', '2018.0', '2016.01', '2019.0.1', '2019']),\n",
    "                     'PERFORMANCE': pd.api.types.CategoricalDtype(['POLE', 'PEDESTRIAN', 'UNKNOWN', 'ECE', 'FRONT', 'RCAR', 'COCKPIT', 'OVERSLAM', 'REAR', 'SIDE', 'WHEEL']),\n",
    "                    'MPLINK': pd.api.types.CategoricalDtype(['YES', 'NO']),\n",
    "                    'NTNU': pd.api.types.CategoricalDtype(['NO', 'YES']),\n",
    "                    'MPLINK_NTNU': pd.api.types.CategoricalDtype(['NO', 'YES']),\n",
    "                    'MBS': pd.api.types.CategoricalDtype(['NO', 'USED']),\n",
    "                    'CLUSTER':pd.api.types.CategoricalDtype(['HPC3', 'HPC1', 'HPC2'])}\n",
    "                                                              \n",
    "TARGET_COLUMN = 'ELAPSEDTIME'\n",
    "QUERY = '''SELECT \n",
    "       VERSION, PERFORMANCE, PRECISION, MPLINK, NTNU, MPLINK_NTNU,\n",
    "       MBS, RUNEND, TIMESTEP, NBNODES, NBELEM1D, NBELEM2D,\n",
    "       NBELEM3D, CLUSTER, NBSERVERS, NBCORE, DATACHECK_TIME,\n",
    "       ELAPSEDTIME\n",
    "FROM\n",
    "  `challenge.training_data`'''\n",
    "\n",
    "BATCH_SIZE = 128\n",
    "NUM_EPOCHS = 5\n",
    "LEARNING_RATE = 0.001\n",
    "\n",
    "# Read the data from BigQuery\n",
    "client = bigquery.Client(location='EU') \n",
    "query_job = client.query(QUERY)\n",
    "data_df = query_job.to_dataframe()  # you can read from other sources to pandas DataFrame\n",
    "print(f'First rows for the raw dataset: \\n{data_df.head()}')\n",
    "\n",
    "# Convert integer valued (numeric) columns to floating point\n",
    "numeric_columns = data_df.select_dtypes(['int64']).columns\n",
    "data_df[numeric_columns] = data_df[numeric_columns].astype('float32')\n",
    "\n",
    "# Convert categorical columns to numeric\n",
    "cat_columns = data_df.select_dtypes(['object']).columns\n",
    "data_df[cat_columns] = data_df[cat_columns].astype('category')\n",
    "data_df[cat_columns] = data_df[cat_columns].apply(lambda x: x.astype(\n",
    "        CATEGORICAL_TYPES[x.name]))\n",
    "data_df[cat_columns] = data_df[cat_columns].apply(lambda x: x.cat.codes)\n",
    "print(f'First rows for the transformed dataset: \\n{data_df.head()}')\n",
    "\n",
    "# Train/Val split\n",
    "train_df, val_df = train_test_split(data_df, train_size=0.8)\n",
    "train_target = train_df.pop(TARGET_COLUMN)\n",
    "val_target = val_df.pop(TARGET_COLUMN)\n",
    "num_train_examples = len(train_df)\n",
    "num_val_examples = len(val_df)\n",
    "\n",
    "# *Possible improvements*: add standartization for numeric values to range [-1; 1], categories to one-hot encoded\n",
    "\n",
    "# Creata tensorflow dataset object\n",
    "dataset_train = (tf.data.Dataset\n",
    "                 .from_tensor_slices((train_df.to_dict('list'), train_target))\n",
    "                 .shuffle(buffer_size=BATCH_SIZE*4)\n",
    "                 .repeat()\n",
    "                 .batch(BATCH_SIZE))\n",
    "                 \n",
    "dataset_val = (tf.data.Dataset\n",
    "                 .from_tensor_slices((val_df.to_dict('list'), val_target))\n",
    "                 .repeat()\n",
    "                 .batch(BATCH_SIZE))  # No shuffle\n",
    "\n",
    "print(f'One batch of the train data:\\n {next(iter(dataset_train))}')\n",
    "\n",
    "# Prepare named inputs for our model\n",
    "inputs = {key: tf.keras.layers.Input(shape=(), name=key) for key in train_df.keys()}\n",
    "x = tf.stack(list(inputs.values()), axis=-1)\n",
    "\n",
    "# Define model's architecture\n",
    "x = tf.keras.layers.Dense(100, activation='relu')(x)\n",
    "x = tf.keras.layers.Dense(50, activation='relu')(x)\n",
    "x = tf.keras.layers.Dense(10, activation='relu')(x)\n",
    "output = tf.keras.layers.Dense(1, activation='linear')(x)\n",
    "\n",
    "# Build the model and compile it\n",
    "model_func = tf.keras.Model(inputs=inputs, outputs=output)\n",
    "model_func.compile(optimizer=tf.keras.optimizers.Adam(LEARNING_RATE),\n",
    "                   loss='mean_absolute_error')\n",
    "\n",
    "# Train the model\n",
    "history = model_func.fit(dataset_train, \n",
    "                          epochs=NUM_EPOCHS, \n",
    "                          steps_per_epoch=int(num_train_examples/BATCH_SIZE), \n",
    "                          validation_data=dataset_val, \n",
    "                          validation_steps=int(num_train_examples/BATCH_SIZE), \n",
    "                          verbose=1)\n",
    "\n",
    "model_func.save(f'{JOB_DIR}/export/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "jur6q1OPOKyx",
    "outputId": "0b371fe5-301c-474a-a768-cc5d29aab790"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./setup.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./setup.py\n",
    "from setuptools import find_packages\n",
    "from setuptools import setup\n",
    "\n",
    "REQUIRED_PACKAGES = ['scikit-learn', 'pandas']\n",
    "\n",
    "setup(\n",
    "    name='trainer',\n",
    "    version='0.1',\n",
    "    install_requires=REQUIRED_PACKAGES,\n",
    "    packages=find_packages(),\n",
    "    include_package_data=True,\n",
    "    description='My super training application package.'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "c2aZrtEpOKyy"
   },
   "source": [
    "## Run a local training task\n",
    "This is a good test before your will try to train the model in the cloud"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FzEAON-YOKyy"
   },
   "source": [
    "Many commands we are going to use accept a parameter for setting a region.  \n",
    "A region is a group of Google Cloud data centers used to run computing tasks.  \n",
    "To reduce latency, let's set a variable with a close by data center:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "LzwTIu5YOKyy"
   },
   "outputs": [],
   "source": [
    "REGION = 'europe-west1'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JtNY767rOKyz"
   },
   "source": [
    "For interacting with Cloud AI Platform we are going to use the [gcloud](https://cloud.google.com/sdk/gcloud) command line tool.  \n",
    "Gcloud also provides [properties](https://cloud.google.com/sdk/docs/properties) used by other services.  \n",
    "Let's set the __ml_engine/local_python__ property so AI Platform knows which Python version to use for local training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "To-Lp-rwOKyz",
    "outputId": "200613d1-74d6-4b77-ef0e-96e648123b9a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated property [ml_engine/local_python].\n"
     ]
    }
   ],
   "source": [
    "# Explicitly tell `gcloud ai-platform local train` to use Python 3 \n",
    "!gcloud config set ml_engine/local_python $(which python3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uhbRp9tJOKyz"
   },
   "source": [
    "For submitting a local training job to AI Platform you need to provide:\n",
    "* A directory to store the model and logs: here we are going to use our Google Cloud Storage bucket we created earlier\n",
    "* The path to your trainer package\n",
    "* The name of your trainer module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "Ew_A_jr7OKy0",
    "outputId": "3c13e3ba-7e97-4e22-8db0-c6914c219e6d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_job_2021_02_04_055807\n",
      "gs://ml-competition-youripn/training_job_2021_02_04_055807\n"
     ]
    }
   ],
   "source": [
    "# Define a timestamped job name\n",
    "JOB_NAME = f\"training_job_{datetime.now().strftime('%Y_%m_%d_%H%M%S')}\"; print(JOB_NAME)\n",
    "JOB_DIR = f'gs://{BUCKET_NAME}/{JOB_NAME}'; print(JOB_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tMVitxfVOKy0"
   },
   "source": [
    "Let's run our local training job with the gcloud command ```gcloud ai-platform local train```."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "ayXRyxxnOKy0",
    "outputId": "8e2c6e44-da2e-430a-f9f9-5952c284d96c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/google/auth/_default.py:69: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. We recommend you rerun `gcloud auth application-default login` and make sure a quota project is added. Or you can use service accounts instead. For more information about service accounts, see https://cloud.google.com/docs/authentication/\n",
      "  warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n",
      "/opt/conda/lib/python3.7/site-packages/google/auth/_default.py:69: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. We recommend you rerun `gcloud auth application-default login` and make sure a quota project is added. Or you can use service accounts instead. For more information about service accounts, see https://cloud.google.com/docs/authentication/\n",
      "  warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n",
      "First rows for the raw dataset: \n",
      "    VERSION PERFORMANCE  PRECISION  ... NBCORE DATACHECK_TIME ELAPSEDTIME\n",
      "0  2018.0.1        POLE          1  ...    288         446.00       15000\n",
      "1  2018.0.1        POLE          1  ...    288         446.00       15600\n",
      "2  2018.0.1        POLE          1  ...    288         446.00       14600\n",
      "3  2018.0.1  PEDESTRIAN          1  ...     36          74.70        4510\n",
      "4   2016.05     UNKNOWN          1  ...     48           4.42        4060\n",
      "\n",
      "[5 rows x 18 columns]\n",
      "First rows for the transformed dataset: \n",
      "   VERSION  PERFORMANCE  PRECISION  ...  NBCORE  DATACHECK_TIME  ELAPSEDTIME\n",
      "0        0            0        1.0  ...   288.0          446.00      15000.0\n",
      "1        0            0        1.0  ...   288.0          446.00      15600.0\n",
      "2        0            0        1.0  ...   288.0          446.00      14600.0\n",
      "3        0            1        1.0  ...    36.0           74.70       4510.0\n",
      "4        1            2        1.0  ...    48.0            4.42       4060.0\n",
      "\n",
      "[5 rows x 18 columns]\n",
      "\n",
      "User settings:\n",
      "\n",
      "   KMP_AFFINITY=granularity=fine,verbose,compact,1,0\n",
      "   KMP_BLOCKTIME=0\n",
      "   KMP_DUPLICATE_LIB_OK=True\n",
      "   KMP_INIT_AT_FORK=FALSE\n",
      "   KMP_SETTINGS=1\n",
      "   OMP_NUM_THREADS=1\n",
      "\n",
      "Effective settings:\n",
      "\n",
      "   KMP_ABORT_DELAY=0\n",
      "   KMP_ADAPTIVE_LOCK_PROPS='1,1024'\n",
      "   KMP_ALIGN_ALLOC=64\n",
      "   KMP_ALL_THREADPRIVATE=128\n",
      "   KMP_ATOMIC_MODE=2\n",
      "   KMP_BLOCKTIME=0\n",
      "   KMP_CPUINFO_FILE: value is not defined\n",
      "   KMP_DETERMINISTIC_REDUCTION=false\n",
      "   KMP_DEVICE_THREAD_LIMIT=2147483647\n",
      "   KMP_DISP_HAND_THREAD=false\n",
      "   KMP_DISP_NUM_BUFFERS=7\n",
      "   KMP_DUPLICATE_LIB_OK=true\n",
      "   KMP_FORCE_REDUCTION: value is not defined\n",
      "   KMP_FOREIGN_THREADS_THREADPRIVATE=true\n",
      "   KMP_FORKJOIN_BARRIER='2,2'\n",
      "   KMP_FORKJOIN_BARRIER_PATTERN='hyper,hyper'\n",
      "   KMP_FORKJOIN_FRAMES=true\n",
      "   KMP_FORKJOIN_FRAMES_MODE=3\n",
      "   KMP_GTID_MODE=3\n",
      "   KMP_HANDLE_SIGNALS=false\n",
      "   KMP_HOT_TEAMS_MAX_LEVEL=1\n",
      "   KMP_HOT_TEAMS_MODE=0\n",
      "   KMP_INIT_AT_FORK=true\n",
      "   KMP_ITT_PREPARE_DELAY=0\n",
      "   KMP_LIBRARY=throughput\n",
      "   KMP_LOCK_KIND=queuing\n",
      "   KMP_MALLOC_POOL_INCR=1M\n",
      "   KMP_MWAIT_HINTS=0\n",
      "   KMP_NUM_LOCKS_IN_BLOCK=1\n",
      "   KMP_PLAIN_BARRIER='2,2'\n",
      "   KMP_PLAIN_BARRIER_PATTERN='hyper,hyper'\n",
      "   KMP_REDUCTION_BARRIER='1,1'\n",
      "   KMP_REDUCTION_BARRIER_PATTERN='hyper,hyper'\n",
      "   KMP_SCHEDULE='static,balanced;guided,iterative'\n",
      "   KMP_SETTINGS=true\n",
      "   KMP_SPIN_BACKOFF_PARAMS='4096,100'\n",
      "   KMP_STACKOFFSET=64\n",
      "   KMP_STACKPAD=0\n",
      "   KMP_STACKSIZE=8M\n",
      "   KMP_STORAGE_MAP=false\n",
      "   KMP_TASKING=2\n",
      "   KMP_TASKLOOP_MIN_TASKS=0\n",
      "   KMP_TASK_STEALING_CONSTRAINT=1\n",
      "   KMP_TEAMS_THREAD_LIMIT=1\n",
      "   KMP_TOPOLOGY_METHOD=all\n",
      "   KMP_USER_LEVEL_MWAIT=false\n",
      "   KMP_USE_YIELD=1\n",
      "   KMP_VERSION=false\n",
      "   KMP_WARNINGS=true\n",
      "   OMP_AFFINITY_FORMAT='OMP: pid %P tid %i thread %n bound to OS proc set {%A}'\n",
      "   OMP_ALLOCATOR=omp_default_mem_alloc\n",
      "   OMP_CANCELLATION=false\n",
      "   OMP_DEBUG=disabled\n",
      "   OMP_DEFAULT_DEVICE=0\n",
      "   OMP_DISPLAY_AFFINITY=false\n",
      "   OMP_DISPLAY_ENV=false\n",
      "   OMP_DYNAMIC=false\n",
      "   OMP_MAX_ACTIVE_LEVELS=2147483647\n",
      "   OMP_MAX_TASK_PRIORITY=0\n",
      "   OMP_NESTED=false\n",
      "   OMP_NUM_THREADS='1'\n",
      "   OMP_PLACES: value is not defined\n",
      "   OMP_PROC_BIND='intel'\n",
      "   OMP_SCHEDULE='static'\n",
      "   OMP_STACKSIZE=8M\n",
      "   OMP_TARGET_OFFLOAD=DEFAULT\n",
      "   OMP_THREAD_LIMIT=2147483647\n",
      "   OMP_TOOL=enabled\n",
      "   OMP_TOOL_LIBRARIES: value is not defined\n",
      "   OMP_WAIT_POLICY=PASSIVE\n",
      "   KMP_AFFINITY='verbose,warnings,respect,granularity=fine,compact,1,0'\n",
      "\n",
      "2021-02-04 05:58:20.902390: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2300000000 Hz\n",
      "2021-02-04 05:58:20.902608: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55a37b997540 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2021-02-04 05:58:20.902637: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2021-02-04 05:58:20.903228: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n",
      "One batch of the train data:\n",
      " ({'VERSION': <tf.Tensor: shape=(128,), dtype=int32, numpy=\n",
      "array([1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1,\n",
      "       1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1,\n",
      "       1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1,\n",
      "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "       0, 1, 0, 1, 1, 1, 4, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1,\n",
      "       0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1], dtype=int32)>, 'PERFORMANCE': <tf.Tensor: shape=(128,), dtype=int32, numpy=\n",
      "array([ 2,  4,  2,  8,  2,  1,  2,  8,  4,  7,  2,  6,  3,  5,  1,  3,  4,\n",
      "        2,  4,  2,  2,  2,  4,  2,  2,  2,  2,  2,  2,  1,  1,  1,  2,  1,\n",
      "        1,  4,  4,  4,  9,  1,  1,  1,  9,  2,  2, 10,  2,  7,  2,  1,  2,\n",
      "        3,  4,  4,  2,  2,  2,  2,  2,  1,  2,  1,  9,  1,  4, 10,  2,  2,\n",
      "        2,  2,  2,  1,  3,  2,  2,  4,  9,  2,  2,  2,  1, 10,  4,  2,  3,\n",
      "        1,  1,  2,  6,  3,  4,  0,  1,  1,  1,  8,  1,  2,  2,  2,  2,  0,\n",
      "        1,  3,  9,  2,  2,  2,  2,  0,  4,  4,  2,  1,  3,  3,  2,  2,  4,\n",
      "        9,  3,  1,  2,  2,  2,  4,  2,  4], dtype=int32)>, 'PRECISION': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 2., 1., 2., 1., 1., 2.,\n",
      "       1., 1., 1., 1., 1., 1., 1., 2., 1., 1., 1., 1., 1., 1., 2., 1., 1.,\n",
      "       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 2., 1., 2., 1.,\n",
      "       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "       1., 1., 1., 1., 1., 1., 1., 1., 1.], dtype=float32)>, 'MPLINK': <tf.Tensor: shape=(128,), dtype=int32, numpy=\n",
      "array([0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
      "       0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1,\n",
      "       0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1,\n",
      "       1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
      "       1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0], dtype=int32)>, 'NTNU': <tf.Tensor: shape=(128,), dtype=int32, numpy=\n",
      "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=int32)>, 'MPLINK_NTNU': <tf.Tensor: shape=(128,), dtype=int32, numpy=\n",
      "array([0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,\n",
      "       0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
      "       1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0], dtype=int32)>, 'MBS': <tf.Tensor: shape=(128,), dtype=int32, numpy=\n",
      "array([0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=int32)>, 'RUNEND': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([ 120.01,  210.01,   20.01,  120.01,  400.01,   35.01,  110.01,\n",
      "        150.01,  120.01,  150.1 ,   80.01,  120.  ,  450.  ,  160.01,\n",
      "         50.01,  100.01,  100.01,  220.01,  130.01,   80.01,  200.01,\n",
      "        130.01,  100.01,  150.01,  350.01,   50.  ,  120.01, 1200.  ,\n",
      "        200.01,   30.01,   50.01,   50.01,  300.  ,   29.01,   50.01,\n",
      "        130.01,  200.01,  120.01,  150.01,   42.01,   32.01,   32.01,\n",
      "        150.01,  200.  ,  160.01, 3500.1 ,   80.  ,  100.1 ,  150.1 ,\n",
      "         60.01,  156.1 ,  450.  ,  200.  ,  130.01,  150.  ,  130.01,\n",
      "         80.01,   50.1 ,    8.  ,   50.01,  100.1 ,   34.01,  150.01,\n",
      "         50.01,  120.01, 3500.1 ,  460.01,  300.  ,  150.  ,  250.01,\n",
      "         85.  ,   32.01,  300.01,  200.  ,  150.01,  120.01,   80.01,\n",
      "         50.01,   90.01,   50.01,   30.01, 3500.1 ,  120.01,   60.1 ,\n",
      "        300.  ,   32.01,   50.01,  300.  ,  100.01,  300.01,  120.01,\n",
      "        110.01,   50.01,   34.01,   50.01,  150.01,   28.01,  150.01,\n",
      "        300.  ,  150.01,  160.01,  110.01,   50.01,  200.1 ,  150.01,\n",
      "        110.01,   80.01,   50.  ,   50.01,  110.01,  100.  ,  150.01,\n",
      "        130.01,   50.01,  300.01,  200.01,  150.01, 1200.  ,   15.01,\n",
      "        140.01,  300.  ,   50.01,  450.  ,  300.  ,  160.01,  120.01,\n",
      "        130.01,  130.01], dtype=float32)>, 'TIMESTEP': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 4.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 1.0e-03,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 7.0e-04, 5.0e-04, 1.5e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 4.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 4.0e-05, 5.0e-04, 4.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 1.5e-04, 5.0e-04, 5.0e-04, 3.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 3.0e-04, 5.0e-04, 7.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 1.5e-04, 5.0e-04, 4.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 3.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       7.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 7.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 1.0e-03, 7.0e-04,\n",
      "       5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 5.0e-04, 1.8e-04, 5.0e-04,\n",
      "       5.0e-04, 5.0e-04], dtype=float32)>, 'NBNODES': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([5190521., 7752571., 1402070., 3890030., 2358462., 1328635.,\n",
      "       8257453., 5311864., 7056899.,  854929.,  715528.,  339267.,\n",
      "       1832689., 5672386., 1337613., 1683670., 5698313., 1257144.,\n",
      "       6660369., 6964357., 4515076., 5239571., 5221789., 4433556.,\n",
      "       1413649., 1327091., 6413849.,  278194., 4546780., 2102504.,\n",
      "       1744399., 1196854.,  366342., 1511298., 1194210., 6045894.,\n",
      "       5308871., 9016911., 5713344., 2693067., 2644107., 2450787.,\n",
      "       6663006.,  370562., 6335280.,  819812.,  448710.,  726547.,\n",
      "        335929., 2253228.,  536782., 1754082., 8945163., 4909360.,\n",
      "        452656., 1659787.,   72599., 1596925., 1076374., 1505982.,\n",
      "        437497., 1685559., 4916748., 1331175., 3779969.,  973964.,\n",
      "       1723297.,  366342.,  379990.,  847719.,  184565., 2518397.,\n",
      "       1676090.,  394802., 4598295., 4843117., 6438389., 1964449.,\n",
      "       4555246., 2443755., 1535721.,  763907., 6987440.,  843350.,\n",
      "       1639086., 2673963., 1494258.,  319822.,  706004., 1753645.,\n",
      "       7093835., 4507822., 1860872., 1685559., 1170337., 4726311.,\n",
      "       2650071.,  731818.,  381160., 4297461., 6522841., 4506704.,\n",
      "       2162000.,  826195., 4543309., 6029474.,  868244., 1444445.,\n",
      "       2440884., 4497564., 8945201., 6593886., 5517186., 1151257.,\n",
      "        179825., 1716277., 2999045.,  390297., 4338451., 6574179.,\n",
      "        281058., 1199206.,  203548.,  366342., 5177972., 5819338.,\n",
      "       1859701., 6069103.], dtype=float32)>, 'NBELEM1D': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([2.99810e+04, 2.13620e+04, 3.65500e+03, 1.03990e+04, 5.72960e+04,\n",
      "       2.73200e+03, 1.85020e+04, 1.96000e+04, 2.99390e+04, 2.20600e+03,\n",
      "       2.67300e+03, 7.43000e+02, 1.34870e+04, 1.83180e+04, 4.61200e+03,\n",
      "       9.01500e+03, 1.30790e+04, 3.50400e+03, 1.95850e+04, 2.35850e+04,\n",
      "       1.56310e+04, 1.79120e+04, 1.35240e+04, 1.40800e+04, 7.96000e+03,\n",
      "       2.86900e+03, 1.18980e+04, 5.95000e+02, 1.50720e+04, 5.95300e+03,\n",
      "       5.03200e+03, 2.21700e+03, 2.65400e+03, 4.38700e+03, 2.22800e+03,\n",
      "       2.27530e+04, 2.71760e+04, 2.15320e+04, 1.81240e+04, 6.11100e+03,\n",
      "       6.20700e+03, 6.10800e+03, 2.81330e+04, 2.00400e+03, 1.92340e+04,\n",
      "       8.31120e+04, 3.18300e+03, 2.26300e+03, 5.04500e+03, 5.89500e+03,\n",
      "       1.34700e+03, 1.06100e+04, 2.33180e+04, 2.15650e+04, 2.28200e+03,\n",
      "       5.81700e+03, 3.00000e+00, 3.10100e+03, 1.00000e+00, 4.39000e+03,\n",
      "       1.77600e+03, 3.84700e+03, 1.87200e+04, 4.33300e+03, 1.32100e+04,\n",
      "       1.00719e+05, 5.31100e+03, 2.65400e+03, 2.88800e+03, 2.15300e+03,\n",
      "       5.42000e+02, 6.77100e+03, 5.83200e+03, 2.63200e+03, 1.38340e+04,\n",
      "       1.83730e+04, 2.53620e+04, 2.30000e+03, 1.35050e+04, 5.74400e+03,\n",
      "       2.71000e+03, 7.31950e+04, 1.86080e+04, 2.15700e+03, 6.56800e+03,\n",
      "       6.20600e+03, 4.38700e+03, 2.89800e+03, 2.36600e+03, 1.06060e+04,\n",
      "       2.99180e+04, 1.56950e+04, 5.23100e+03, 3.84700e+03, 2.33800e+03,\n",
      "       1.62380e+04, 6.20200e+03, 1.14700e+03, 2.78900e+03, 1.26860e+04,\n",
      "       2.16570e+04, 1.45770e+04, 6.45800e+03, 2.21000e+03, 1.74450e+04,\n",
      "       1.85990e+04, 2.28900e+03, 3.11100e+03, 5.74400e+03, 1.45550e+04,\n",
      "       2.33560e+04, 2.54950e+04, 1.85710e+04, 4.19900e+03, 3.03300e+03,\n",
      "       9.61500e+03, 7.92300e+03, 1.52000e+03, 1.01210e+04, 2.54170e+04,\n",
      "       3.24800e+03, 2.22800e+03, 2.41500e+03, 2.65400e+03, 2.96690e+04,\n",
      "       1.96420e+04, 6.39300e+03, 2.55310e+04], dtype=float32)>, 'NBELEM2D': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([4704880., 7698143., 1413142., 3700417., 2274923., 1346708.,\n",
      "       8101128., 5205459., 6344012.,  826921.,  696052.,  100866.,\n",
      "       1633318., 5518268., 1023093., 1606294., 5517562., 1265764.,\n",
      "       6575588., 5753601., 4342083., 4979890., 4740532., 4429457.,\n",
      "       1369665., 1356377., 5623819.,   40940., 4208821., 2138564.,\n",
      "       1513563., 1230630.,  202473., 1218895., 1227082., 5694707.,\n",
      "       4766047., 8423815., 5854163., 2443462., 2367412., 2167560.,\n",
      "       6719820.,  126242., 5427569.,  292993.,  247199.,  701769.,\n",
      "        294590., 1865321.,  520151., 1684922., 7840617., 4716973.,\n",
      "        268602., 1593253.,   64184., 1638795.,       0., 1219217.,\n",
      "        422931., 1542569., 5047033., 1026450., 3805159.,  340911.,\n",
      "       1674294.,  202473.,  259788.,  849542.,   93868., 2237436.,\n",
      "       1630262.,  242187., 4451122., 4918038., 6469673., 1518010.,\n",
      "       4082165., 2369188., 1535962.,  301106., 6687711.,  817724.,\n",
      "       1585303., 2398604., 1227864.,  197476.,  436858., 1684624.,\n",
      "       6390191., 4036377., 1606909., 1542569., 1192308., 4651658.,\n",
      "       2373920.,  662342.,  262192., 4284948., 5579514., 4036377.,\n",
      "       1779858.,  805943., 4545344., 5734854.,  856571., 1486881.,\n",
      "       2369292., 4027525., 7840617., 5819786., 5234221.,  868450.,\n",
      "        163449., 1571199., 2903317.,   91127., 3761505., 6352078.,\n",
      "        280919., 1232064.,  158678.,  202473., 4642161., 5204508.,\n",
      "       1807291., 6032638.], dtype=float32)>, 'NBELEM3D': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([2.090914e+06, 4.311880e+05, 1.217430e+05, 9.930260e+05,\n",
      "       7.388340e+05, 1.549600e+04, 8.020910e+05, 6.679270e+05,\n",
      "       2.656232e+06, 2.317000e+03, 9.231000e+03, 3.100160e+05,\n",
      "       3.851240e+05, 1.044260e+06, 2.093090e+05, 2.923070e+05,\n",
      "       4.097420e+05, 8.470000e+04, 4.057120e+05, 2.268055e+06,\n",
      "       1.109311e+06, 7.856890e+05, 9.903870e+05, 4.533340e+05,\n",
      "       2.519010e+05, 1.232900e+04, 1.121901e+06, 3.180370e+05,\n",
      "       1.314864e+06, 4.664000e+04, 2.105900e+05, 2.398600e+04,\n",
      "       3.794830e+05, 3.299700e+05, 2.398600e+04, 1.150655e+06,\n",
      "       1.124436e+06, 1.671052e+06, 8.320900e+05, 2.845810e+05,\n",
      "       3.124670e+05, 3.130230e+05, 1.229489e+06, 4.476580e+05,\n",
      "       1.528510e+06, 7.113850e+05, 3.149920e+05, 1.371400e+04,\n",
      "       3.281980e+05, 7.126420e+05, 1.200000e+02, 3.867060e+05,\n",
      "       3.379225e+06, 1.061622e+06, 3.049600e+05, 3.174800e+04,\n",
      "       5.866000e+03, 1.234300e+04, 7.001890e+05, 3.060660e+05,\n",
      "       9.810000e+02, 5.475320e+05, 7.381220e+05, 2.950500e+05,\n",
      "       5.113550e+05, 1.240354e+06, 1.517700e+04, 3.794830e+05,\n",
      "       2.452200e+05, 5.302000e+03, 9.745800e+04, 2.881080e+05,\n",
      "       1.630780e+05, 2.308320e+05, 6.071640e+05, 5.227600e+05,\n",
      "       1.024797e+06, 5.145010e+05, 1.394537e+06, 3.494870e+05,\n",
      "       2.436390e+05, 1.159188e+06, 1.745846e+06, 9.560000e+02,\n",
      "       2.468840e+05, 3.124670e+05, 2.182270e+05, 2.432330e+05,\n",
      "       3.821650e+05, 3.867060e+05, 2.335184e+06, 1.571025e+06,\n",
      "       2.216960e+05, 5.475320e+05, 1.483500e+04, 5.382760e+05,\n",
      "       3.124670e+05, 8.610600e+04, 2.452200e+05, 5.894620e+05,\n",
      "       2.736510e+06, 1.571025e+06, 7.533020e+05, 1.151000e+03,\n",
      "       7.429120e+05, 9.800520e+05, 7.431000e+03, 1.498500e+04,\n",
      "       3.373330e+05, 1.571025e+06, 3.379225e+06, 2.259810e+06,\n",
      "       8.131690e+05, 2.105680e+05, 2.364780e+05, 3.769580e+05,\n",
      "       3.717000e+03, 3.518000e+05, 1.907383e+06, 1.334720e+06,\n",
      "       2.774100e+05, 2.398600e+04, 2.355230e+05, 3.794830e+05,\n",
      "       2.185548e+06, 1.974437e+06, 6.960000e+02, 6.904940e+05],\n",
      "      dtype=float32)>, 'CLUSTER': <tf.Tensor: shape=(128,), dtype=int32, numpy=\n",
      "array([0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0,\n",
      "       0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1,\n",
      "       0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0,\n",
      "       1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1,\n",
      "       1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0,\n",
      "       0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0], dtype=int32)>, 'NBSERVERS': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([5., 4., 2., 3., 5., 2., 5., 5., 6., 2., 2., 2., 2., 5., 2., 2., 5.,\n",
      "       2., 4., 5., 3., 3., 6., 5., 2., 2., 3., 2., 5., 2., 2., 2., 2., 2.,\n",
      "       2., 6., 6., 6., 3., 2., 2., 2., 3., 2., 3., 5., 2., 2., 2., 2., 2.,\n",
      "       2., 6., 6., 2., 2., 2., 2., 2., 2., 2., 2., 3., 2., 6., 5., 2., 2.,\n",
      "       2., 2., 2., 2., 2., 2., 5., 4., 5., 2., 5., 2., 2., 5., 6., 2., 2.,\n",
      "       2., 2., 2., 2., 2., 6., 3., 2., 2., 2., 5., 2., 2., 2., 3., 5., 3.,\n",
      "       2., 2., 3., 5., 2., 2., 2., 5., 4., 4., 5., 2., 2., 2., 2., 2., 2.,\n",
      "       5., 2., 2., 2., 2., 5., 4., 2., 4.], dtype=float32)>, 'NBCORE': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([180., 144.,  48., 108., 180.,  48., 180., 180., 216.,  48.,  48.,\n",
      "        48.,  48., 180.,  48.,  48., 180.,  48., 144., 180., 108., 108.,\n",
      "       216., 180.,  48.,  48., 108.,  48., 180.,  48.,  48.,  48.,  48.,\n",
      "        48.,  48., 216., 216., 216., 108.,  48.,  48.,  48., 108.,  48.,\n",
      "       108., 180.,  48.,  48.,  48.,  48.,  48.,  48., 216., 216.,  48.,\n",
      "        48.,  48.,  48.,  48.,  48.,  48.,  48., 108.,  48., 216., 180.,\n",
      "        48.,  48.,  48.,  48.,  48.,  48.,  48.,  48., 180., 144., 180.,\n",
      "        48., 180.,  48.,  48., 180., 216.,  48.,  48.,  48.,  48.,  48.,\n",
      "        48.,  48., 216., 108.,  48.,  48.,  48., 180.,  48.,  48.,  48.,\n",
      "       108., 180., 108.,  48.,  48., 108., 180.,  48.,  48.,  48., 180.,\n",
      "       144., 144., 180.,  48.,  48.,  48.,  48.,  48.,  48., 180.,  48.,\n",
      "        48.,  48.,  48., 180., 144.,  48., 144.], dtype=float32)>, 'DATACHECK_TIME': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([ 462.  ,  706.  ,   61.1 ,  317.  ,  104.  ,   67.4 ,  946.  ,\n",
      "        457.  ,  711.  ,   25.1 ,   19.4 ,   13.9 ,  122.  ,  512.  ,\n",
      "         81.6 ,  114.  ,  395.  ,   50.6 ,  437.  ,  830.  ,  360.  ,\n",
      "       1390.  ,  346.  ,  360.  ,   52.3 ,   49.8 ,  381.  ,   16.2 ,\n",
      "        426.  ,   84.5 ,   89.3 ,   46.3 ,   56.  ,  109.  ,   47.4 ,\n",
      "        500.  ,  496.  ,  742.  ,  519.  ,  152.  ,  182.  ,  190.  ,\n",
      "        656.  ,   19.2 ,  619.  ,   29.4 ,   22.8 ,   27.  ,   13.7 ,\n",
      "        132.  ,   17.3 ,   92.1 ,  731.  ,  452.  ,   18.4 ,  100.  ,\n",
      "          4.84,   65.2 ,   33.6 ,  111.  ,   16.3 ,   92.1 ,  539.  ,\n",
      "         78.6 ,  224.  ,   38.6 ,  104.  ,   55.7 ,   50.2 ,   33.8 ,\n",
      "          7.05,  198.  ,   71.6 ,   15.2 ,  485.  ,  332.  ,  715.  ,\n",
      "         51.6 ,  274.  ,  143.  ,   68.5 ,   31.4 ,  635.  ,   43.5 ,\n",
      "         83.3 ,  170.  ,  101.  ,   16.5 ,   75.1 ,   91.6 ,  751.  ,\n",
      "        462.  ,   95.9 ,   95.6 ,   33.5 ,  391.  ,  195.  ,   20.1 ,\n",
      "         17.5 ,  349.  ,  761.  ,  459.  ,  142.  ,   27.4 ,  426.  ,\n",
      "        555.  ,   28.2 ,   64.8 ,  143.  ,  452.  ,  749.  ,  594.  ,\n",
      "        612.  ,   65.2 ,    9.74,  159.  ,  103.  ,   20.2 ,  177.  ,\n",
      "        660.  ,   10.4 ,   46.8 ,   11.2 ,   58.7 ,  447.  ,  502.  ,\n",
      "         71.4 ,  887.  ], dtype=float32)>}, <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
      "array([ 32000.,  60300.,   2170.,  23300.,  17900.,   2760.,  25500.,\n",
      "        28200.,  41700.,  11600.,   4240.,   5950.,  90100.,  21800.,\n",
      "         5130.,  15100.,  14900.,  18200.,  28800.,  25000.,  41700.,\n",
      "        28600.,  37000.,  16200.,  35800.,   3870.,  21600.,  15300.,\n",
      "        40300.,   3520.,   7410.,   4660.,  15700.,   3790.,   4650.,\n",
      "        29400.,  41800.,  40700.,  37100.,   8870.,   6840.,   6130.,\n",
      "        46600.,   5720.,  54600.,  97100.,   4300.,   8710.,   3540.,\n",
      "         9590.,  13600.,  54300.,  70900.,  22400.,   4460.,  12600.,\n",
      "         6300.,   4590.,   4250.,   6060.,   3820.,   3220.,  29300.,\n",
      "         5080.,  12900., 102000.,  57100.,  15000.,  12200.,  18300.,\n",
      "         2760.,   7230.,  36400.,  16300.,  25200.,  22100.,  20100.,\n",
      "         5950.,  13100.,  11700.,   2860., 128000.,  36400.,   7820.,\n",
      "        39200.,   6390.,   5990.,  21700.,   8450.,  37900.,  40900.,\n",
      "        22000.,   7930.,   3660.,   3750.,  19800.,   6080.,   6330.,\n",
      "        15100.,  23200.,  44800.,  23700.,   6310.,  10800.,  52700.,\n",
      "        20700.,   4450.,   5950.,  11400.,  20200.,  51000.,  54100.,\n",
      "        21100.,   5310.,   4210.,  31800.,  27600.,  29300.,   4280.,\n",
      "        30600.,   9190.,   4880.,  35100.,  17700.,  67500.,  24900.,\n",
      "        16300.,  29500.], dtype=float32)>)\n",
      "Epoch 1/5\n",
      "OMP: Info #212: KMP_AFFINITY: decoding x2APIC ids.\n",
      "OMP: Info #210: KMP_AFFINITY: Affinity capable, using global cpuid leaf 11 info\n",
      "OMP: Info #154: KMP_AFFINITY: Initial OS proc set respected: 0\n",
      "OMP: Info #156: KMP_AFFINITY: 1 available OS procs\n",
      "OMP: Info #157: KMP_AFFINITY: Uniform topology\n",
      "OMP: Info #159: KMP_AFFINITY: 1 packages x 1 cores/pkg x 1 threads/core (1 total cores)\n",
      "OMP: Info #214: KMP_AFFINITY: OS proc to physical thread map:\n",
      "OMP: Info #171: KMP_AFFINITY: OS proc 0 maps to package 0 \n",
      "OMP: Info #250: KMP_AFFINITY: pid 18433 tid 18494 thread 0 bound to OS proc set 0\n",
      "OMP: Info #250: KMP_AFFINITY: pid 18433 tid 18494 thread 1 bound to OS proc set 0\n",
      "OMP: Info #250: KMP_AFFINITY: pid 18433 tid 18493 thread 2 bound to OS proc set 0\n",
      "556/556 [==============================] - 5s 10ms/step - loss: 12769.9033 - val_loss: 10249.0469\n",
      "Epoch 2/5\n",
      "556/556 [==============================] - 5s 8ms/step - loss: 11010.7988 - val_loss: 9091.9033\n",
      "Epoch 3/5\n",
      "556/556 [==============================] - 4s 8ms/step - loss: 10398.7715 - val_loss: 9008.3496\n",
      "Epoch 4/5\n",
      "556/556 [==============================] - 5s 8ms/step - loss: 9955.1953 - val_loss: 9741.7725\n",
      "Epoch 5/5\n",
      "556/556 [==============================] - 5s 8ms/step - loss: 9941.4912 - val_loss: 8648.6055\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow/python/training/tracking/tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "2021-02-04 05:58:49.842468: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow/python/training/tracking/tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n"
     ]
    }
   ],
   "source": [
    "# Run the localtraining job\n",
    "! gcloud ai-platform local train \\\n",
    "  --job-dir $JOB_DIR \\\n",
    "  --package-path ./trainer \\\n",
    "  --module-name trainer.task "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SVET52ULOKy1"
   },
   "source": [
    "Your model has been saved to your Google Cloud Storage bucket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "bpjLdsCcOKy1",
    "outputId": "291e12e7-3b8c-4c1a-f6d2-70de352e3e36"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://ml-competition-youripn/latest_model/\n",
      "gs://ml-competition-youripn/training_job_2021_02_03_165524/\n"
     ]
    }
   ],
   "source": [
    "!gsutil ls gs://{BUCKET_NAME}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hot8GNzaOKy2"
   },
   "source": [
    "# AI Platform training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "U8ktaOw8OKy2"
   },
   "source": [
    "Train your model in the cloud.  \n",
    "This can help when you'll need more compute power, run your training for a long periods of time or try several trainings in parallel with hyperparameters search."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zXFJu_pHOKy2"
   },
   "source": [
    "When training through AI Platform you need a few more parameters:\n",
    "* __Region__: the region used by AI Platform for training\n",
    "* __Runtime version__: the AI Platform version you want to use\n",
    "* __Python version__: the Python version used by your package\n",
    "* __Scale tier__: define which compute power will be used (GPU, TPU, number of machines, ...), more details in [this documentation](https://cloud.google.com/ai-platform/training/docs/machine-types)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TXZZdq6KOKy2"
   },
   "source": [
    "Let's submit a training job with ```gcloud ai-platform jobs submit training``` with a basic configuration (only 1 machine, no GPU, no TPU):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "xO20c8xyOKy2",
    "outputId": "bd0a70d9-a57b-46ca-960e-823d2d18d981"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job [training_job_2021_02_04_055807] submitted successfully.\n",
      "Your job is still active. You may view the status of your job with the command\n",
      "\n",
      "  $ gcloud ai-platform jobs describe training_job_2021_02_04_055807\n",
      "\n",
      "or continue streaming the logs with the command\n",
      "\n",
      "  $ gcloud ai-platform jobs stream-logs training_job_2021_02_04_055807\n",
      "jobId: training_job_2021_02_04_055807\n",
      "state: QUEUED\n"
     ]
    }
   ],
   "source": [
    "# Submit the training job\n",
    "! gcloud ai-platform jobs submit training $JOB_NAME \\\n",
    "  --job-dir $JOB_DIR \\\n",
    "  --package-path ./trainer \\\n",
    "  --module-name trainer.task \\\n",
    "  --region $REGION \\\n",
    "  --runtime-version=2.1 \\\n",
    "  --python-version=3.7 \\\n",
    "  --scale-tier basic "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "a3e2DlqPOKy3"
   },
   "source": [
    "The training job is a long running operation.  \n",
    "You can use ```gcloud ai-platform jobs describe``` to get the status of the job:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "ioMSA6DzOKy3",
    "outputId": "8749237a-660b-4e20-dde6-ec3258f2bc8d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "createTime: '2021-02-04T05:59:01Z'\n",
      "etag: B5PxPh3czmE=\n",
      "jobId: training_job_2021_02_04_055807\n",
      "state: PREPARING\n",
      "trainingInput:\n",
      "  jobDir: gs://ml-competition-youripn/training_job_2021_02_04_055807\n",
      "  packageUris:\n",
      "  - gs://ml-competition-youripn/training_job_2021_02_04_055807/packages/5c587215ffc895507fdf56f1cc8c2e427a079eafdcd0be96d9ac55dbbaa96dfa/trainer-0.1.tar.gz\n",
      "  pythonModule: trainer.task\n",
      "  pythonVersion: '3.7'\n",
      "  region: europe-west1\n",
      "  runtimeVersion: '2.1'\n",
      "trainingOutput: {}\n",
      "\n",
      "View job in the Cloud Console at:\n",
      "https://console.cloud.google.com/mlengine/jobs/training_job_2021_02_04_055807?project=data-science-showroom\n",
      "\n",
      "View logs at:\n",
      "https://console.cloud.google.com/logs?resource=ml_job%2Fjob_id%2Ftraining_job_2021_02_04_055807&project=data-science-showroom\n"
     ]
    }
   ],
   "source": [
    "! gcloud ai-platform jobs describe $JOB_NAME"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B3Nct7d2OKy3"
   },
   "source": [
    "Your model has been saved to your Google Cloud Storage bucket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "q2myS1CtOKy3",
    "outputId": "19cc9d1c-b456-4c66-a8d3-64f34bd2d93c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://ml-competition-youripn/latest_model/\n",
      "gs://ml-competition-youripn/training_job_2021_02_03_165524/\n",
      "gs://ml-competition-youripn/training_job_2021_02_04_055807/\n"
     ]
    }
   ],
   "source": [
    "!gsutil ls gs://{BUCKET_NAME}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vOpjKNj5OKy4"
   },
   "source": [
    "# AI Platform deployment\n",
    "\n",
    "Now that you have trained your model, it's time to make it available for serving predictions.  \n",
    "Google [AI Platform Prediction](https://cloud.google.com/ai-platform/prediction/docs) lets you do just that very easily."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wgPU4UGJOKy4"
   },
   "source": [
    "Let's use ```gsutil ls``` to list your model's file from the Cloud Storage Bucket:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "VdvCBO2SOKy4",
    "outputId": "2bcd0723-592b-408f-ece3-43b392e954c0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       0 B  2021-02-03T16:56:08Z  gs://ml-competition-youripn/latest_model/export/\n",
      "165.48 KiB  2021-02-04T05:58:55Z  gs://ml-competition-youripn/latest_model/export/saved_model.pb\n",
      "                                 gs://ml-competition-youripn/latest_model/export/assets/\n",
      "                                 gs://ml-competition-youripn/latest_model/export/variables/\n",
      "TOTAL: 2 objects, 169448 bytes (165.48 KiB)\n"
     ]
    }
   ],
   "source": [
    "LATEST_MODEL_DIR = f'gs://{BUCKET_NAME}/latest_model/export' \n",
    "!gsutil ls -lh $LATEST_MODEL_DIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bloHk1x0OKy4"
   },
   "source": [
    "Set a **name** and a **version** for this model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "IhmM2Vt3OKy5"
   },
   "outputs": [],
   "source": [
    "MODEL_NAME = 'kers_youripn' # Choose your own model name\n",
    "MODEL_VERSION = 'v1' # Make sure to increase version when deploying a new version of the same model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "07VBcpHDOKy5"
   },
   "source": [
    "Let's use ```gcloud ai-platform models create```to create a new model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "9dqkllvsOKy5",
    "outputId": "0d4a7911-165b-4118-b1b2-12964618a92b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using endpoint [https://europe-west1-ml.googleapis.com/]\n",
      "Created ml engine model [projects/data-science-showroom/models/kers_youripn].\n"
     ]
    }
   ],
   "source": [
    "# create a model object at AI Platform first\n",
    "! gcloud ai-platform models create $MODEL_NAME --region $REGION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JHfsYVaMOKy5"
   },
   "source": [
    "Now that we have the model available in AI Platform, let's create the first version of this model.  \n",
    "We need to point AI Platform to our model in Google Cloud Storage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "WGzjp6J6OKy5",
    "outputId": "5290d381-35d9-417a-a9a4-49596cd04d33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using endpoint [https://europe-west1-ml.googleapis.com/]\n",
      "\u001b[1;31mERROR:\u001b[0m (gcloud.ai-platform.versions.create) PERMISSION_DENIED: Access to model denied.\n"
     ]
    }
   ],
   "source": [
    "# Create model version based on that SavedModel directory\n",
    "! gcloud ai-platform versions create $MODEL_VERSION --region $REGION --model $MODEL_NAME \\\n",
    "  --runtime-version 2.1 \\\n",
    "  --python-version 3.7 \\\n",
    "  --framework tensorflow \\\n",
    "  --origin $LATEST_MODEL_DIR \n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "b0a29mCYOKy6"
   },
   "source": [
    "# Prediction with AI Platform (from a csv file)\n",
    "At this point we have trained a model and made the model available for serving predictions thanks to AI Platform.  \n",
    "Let's get some predictions from this model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "W-s8zQe5OKy6"
   },
   "source": [
    "## Get the test data\n",
    "Let's grab some fresh data to generate predictions on!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "-MI62izWOKy6"
   },
   "outputs": [],
   "source": [
    "%%bigquery data_df\n",
    "SELECT \n",
    "       JOBID, VERSION, PERFORMANCE, PRECISION, MPLINK, NTNU, MPLINK_NTNU,\n",
    "       MBS, RUNEND, TIMESTEP, NBNODES, NBELEM1D, NBELEM2D,\n",
    "       NBELEM3D, CLUSTER, NBSERVERS, NBCORE, DATACHECK_TIME,\n",
    "       ELAPSEDTIME\n",
    "FROM\n",
    "  `challenge.training_data`\n",
    "WHERE MOD(JOBID,4) = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "XiUtQDBUOKy6",
    "outputId": "66e31b52-5d18-44d5-a694-8c403d12e1bf"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>JOBID</th>\n",
       "      <th>VERSION</th>\n",
       "      <th>PERFORMANCE</th>\n",
       "      <th>PRECISION</th>\n",
       "      <th>MPLINK</th>\n",
       "      <th>NTNU</th>\n",
       "      <th>MPLINK_NTNU</th>\n",
       "      <th>MBS</th>\n",
       "      <th>RUNEND</th>\n",
       "      <th>TIMESTEP</th>\n",
       "      <th>NBNODES</th>\n",
       "      <th>NBELEM1D</th>\n",
       "      <th>NBELEM2D</th>\n",
       "      <th>NBELEM3D</th>\n",
       "      <th>CLUSTER</th>\n",
       "      <th>NBSERVERS</th>\n",
       "      <th>NBCORE</th>\n",
       "      <th>DATACHECK_TIME</th>\n",
       "      <th>ELAPSEDTIME</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>974258</td>\n",
       "      <td>2018.0.1</td>\n",
       "      <td>PEDESTRIAN</td>\n",
       "      <td>1</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>40.68</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>1647532</td>\n",
       "      <td>3760</td>\n",
       "      <td>1673587</td>\n",
       "      <td>103822</td>\n",
       "      <td>HPC3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36</td>\n",
       "      <td>74.70</td>\n",
       "      <td>4510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>598282</td>\n",
       "      <td>2016.05</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>1</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>30000.00</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>504</td>\n",
       "      <td>0</td>\n",
       "      <td>410</td>\n",
       "      <td>0</td>\n",
       "      <td>HPC1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>48</td>\n",
       "      <td>0.48</td>\n",
       "      <td>7340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>624046</td>\n",
       "      <td>2016.05</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>1</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>20.00</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>133650</td>\n",
       "      <td>44</td>\n",
       "      <td>127307</td>\n",
       "      <td>0</td>\n",
       "      <td>HPC1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>48</td>\n",
       "      <td>5.86</td>\n",
       "      <td>1870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>625322</td>\n",
       "      <td>2016.05</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>1</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>20.00</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>129912</td>\n",
       "      <td>44</td>\n",
       "      <td>128857</td>\n",
       "      <td>0</td>\n",
       "      <td>HPC1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>48</td>\n",
       "      <td>5.76</td>\n",
       "      <td>1930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>671282</td>\n",
       "      <td>2016.05</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>1</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>120.00</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>72758</td>\n",
       "      <td>228</td>\n",
       "      <td>71208</td>\n",
       "      <td>0</td>\n",
       "      <td>HPC1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>48</td>\n",
       "      <td>3.16</td>\n",
       "      <td>2120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    JOBID   VERSION PERFORMANCE  PRECISION MPLINK NTNU MPLINK_NTNU MBS  \\\n",
       "0  974258  2018.0.1  PEDESTRIAN          1     NO   NO          NO  NO   \n",
       "1  598282   2016.05     UNKNOWN          1     NO   NO          NO  NO   \n",
       "2  624046   2016.05     UNKNOWN          1     NO  YES          NO  NO   \n",
       "3  625322   2016.05     UNKNOWN          1     NO  YES          NO  NO   \n",
       "4  671282   2016.05     UNKNOWN          1    YES   NO          NO  NO   \n",
       "\n",
       "     RUNEND  TIMESTEP  NBNODES  NBELEM1D  NBELEM2D  NBELEM3D CLUSTER  \\\n",
       "0     40.68    0.0005  1647532      3760   1673587    103822    HPC3   \n",
       "1  30000.00    0.0005      504         0       410         0    HPC1   \n",
       "2     20.00    0.0005   133650        44    127307         0    HPC1   \n",
       "3     20.00    0.0005   129912        44    128857         0    HPC1   \n",
       "4    120.00    0.0005    72758       228     71208         0    HPC1   \n",
       "\n",
       "   NBSERVERS  NBCORE  DATACHECK_TIME  ELAPSEDTIME  \n",
       "0        1.0      36           74.70         4510  \n",
       "1        2.0      48            0.48         7340  \n",
       "2        2.0      48            5.86         1870  \n",
       "3        2.0      48            5.76         1930  \n",
       "4        2.0      48            3.16         2120  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "5yqf81laOKy7",
    "outputId": "6a6fca9b-1767-427e-f4d1-3aa68e8a9d03"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First rows for the raw dataset: \n",
      "    JOBID   VERSION PERFORMANCE  PRECISION MPLINK NTNU MPLINK_NTNU MBS  \\\n",
      "0  974258  2018.0.1  PEDESTRIAN          1     NO   NO          NO  NO   \n",
      "1  598282   2016.05     UNKNOWN          1     NO   NO          NO  NO   \n",
      "2  624046   2016.05     UNKNOWN          1     NO  YES          NO  NO   \n",
      "3  625322   2016.05     UNKNOWN          1     NO  YES          NO  NO   \n",
      "4  671282   2016.05     UNKNOWN          1    YES   NO          NO  NO   \n",
      "\n",
      "     RUNEND  TIMESTEP  NBNODES  NBELEM1D  NBELEM2D  NBELEM3D CLUSTER  \\\n",
      "0     40.68    0.0005  1647532      3760   1673587    103822    HPC3   \n",
      "1  30000.00    0.0005      504         0       410         0    HPC1   \n",
      "2     20.00    0.0005   133650        44    127307         0    HPC1   \n",
      "3     20.00    0.0005   129912        44    128857         0    HPC1   \n",
      "4    120.00    0.0005    72758       228     71208         0    HPC1   \n",
      "\n",
      "   NBSERVERS  NBCORE  DATACHECK_TIME  ELAPSEDTIME  \n",
      "0        1.0      36           74.70         4510  \n",
      "1        2.0      48            0.48         7340  \n",
      "2        2.0      48            5.86         1870  \n",
      "3        2.0      48            5.76         1930  \n",
      "4        2.0      48            3.16         2120  \n",
      "First rows for the transformed dataset: \n",
      "      JOBID  VERSION  PERFORMANCE  PRECISION  MPLINK  NTNU  MPLINK_NTNU  MBS  \\\n",
      "0  974258.0        0            1        1.0       1     0            0    0   \n",
      "1  598282.0        1            2        1.0       1     0            0    0   \n",
      "2  624046.0        1            2        1.0       1     1            0    0   \n",
      "3  625322.0        1            2        1.0       1     1            0    0   \n",
      "4  671282.0        1            2        1.0       0     0            0    0   \n",
      "\n",
      "     RUNEND  TIMESTEP    NBNODES  NBELEM1D   NBELEM2D  NBELEM3D  CLUSTER  \\\n",
      "0     40.68    0.0005  1647532.0    3760.0  1673587.0  103822.0        0   \n",
      "1  30000.00    0.0005      504.0       0.0      410.0       0.0        1   \n",
      "2     20.00    0.0005   133650.0      44.0   127307.0       0.0        1   \n",
      "3     20.00    0.0005   129912.0      44.0   128857.0       0.0        1   \n",
      "4    120.00    0.0005    72758.0     228.0    71208.0       0.0        1   \n",
      "\n",
      "   NBSERVERS  NBCORE  DATACHECK_TIME  ELAPSEDTIME  \n",
      "0        1.0    36.0           74.70       4510.0  \n",
      "1        2.0    48.0            0.48       7340.0  \n",
      "2        2.0    48.0            5.86       1870.0  \n",
      "3        2.0    48.0            5.76       1930.0  \n",
      "4        2.0    48.0            3.16       2120.0  \n"
     ]
    }
   ],
   "source": [
    "# Need to avoid \"serving skew\"! \n",
    "# Preprocess test data the same way as we did for training\n",
    "CATEGORICAL_TYPES = {'VERSION': pd.api.types.CategoricalDtype(['2018.0.1', '2016.05', '2016.06', '2019.0.2', '2012.7', '2019.0', '2018.0', '2016.01', '2019.0.1', '2019']),\n",
    "                     'PERFORMANCE': pd.api.types.CategoricalDtype(['POLE', 'PEDESTRIAN', 'UNKNOWN', 'ECE', 'FRONT', 'RCAR', 'COCKPIT', 'OVERSLAM', 'REAR', 'SIDE', 'WHEEL']),\n",
    "                    'MPLINK': pd.api.types.CategoricalDtype(['YES', 'NO']),\n",
    "                    'NTNU': pd.api.types.CategoricalDtype(['NO', 'YES']),\n",
    "                    'MPLINK_NTNU': pd.api.types.CategoricalDtype(['NO', 'YES']),\n",
    "                    'MBS': pd.api.types.CategoricalDtype(['NO', 'USED']),\n",
    "                    'CLUSTER':pd.api.types.CategoricalDtype(['HPC3', 'HPC1', 'HPC2'])}\n",
    "                                                              \n",
    "TARGET_COLUMN = 'ELAPSEDTIME'\n",
    "\n",
    "\n",
    "BATCH_SIZE = 128\n",
    "NUM_EPOCHS = 5\n",
    "LEARNING_RATE = 0.001\n",
    "\n",
    "# Read the data from BigQuery\n",
    "# you can read from other sources to pandas DataFrame\n",
    "print(f'First rows for the raw dataset: \\n{data_df.head()}')\n",
    "\n",
    "# Convert integer valued (numeric) columns to floating point\n",
    "numeric_columns = data_df.select_dtypes(['int64']).columns\n",
    "data_df[numeric_columns] = data_df[numeric_columns].astype('float32')\n",
    "\n",
    "# Convert categorical columns to numeric\n",
    "cat_columns = data_df.select_dtypes(['object']).columns\n",
    "data_df[cat_columns] = data_df[cat_columns].astype('category')\n",
    "data_df[cat_columns] = data_df[cat_columns].apply(lambda x: x.astype(\n",
    "        CATEGORICAL_TYPES[x.name]))\n",
    "data_df[cat_columns] = data_df[cat_columns].apply(lambda x: x.cat.codes)\n",
    "print(f'First rows for the transformed dataset: \\n{data_df.head()}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1CmyfDhqOKy7"
   },
   "source": [
    "### Prepare prediction input file\n",
    "The `gcloud` command-line tool accepts newline-delimited JSON for online\n",
    "prediction, and this particular Keras model expects a flat list of\n",
    "numbers for each input example.\n",
    "\n",
    "AI Platform requires a different format when you make online prediction requests to the REST API without using the `gcloud` tool. The way you structure\n",
    "your model may also change how you must format data for prediction. Learn more\n",
    "about [formatting data for online\n",
    "prediction](https://cloud.google.com/ml-engine/docs/tensorflow/prediction-overview#prediction_input_data)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "S_8IwxvROKy7"
   },
   "source": [
    "Test first on a few samples:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "GwtyPznyOKy7",
    "outputId": "54a9421c-7347-4951-ac96-bca326941541"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'test_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-4af370418df0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mprediction_dict_sample\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'JOBID'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ELAPSEDTIME'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'records'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'prediction_input.json'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mjson_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'test_df' is not defined"
     ]
    }
   ],
   "source": [
    "# Export the prediction input to a JSON file in the format accepted by AI Platform\n",
    "import json\n",
    "\n",
    "prediction_dict_sample = test_df.drop('JOBID', axis=1).drop('ELAPSEDTIME',axis=1)[:5].to_dict('records')\n",
    "\n",
    "with open('prediction_input.json', 'w') as json_file:\n",
    "    json.dump({'instances': prediction_dict_sample}, json_file, indent=' ')\n",
    "\n",
    "! cat prediction_input.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EWhez0G6OKy8"
   },
   "source": [
    "### Test predictions on few samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "taAqvQeNOKy8"
   },
   "source": [
    "Use ```gcloud ai-platform predict``` to generate predictions from your model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nfTZeLa1OKy8",
    "outputId": "54ef4c2f-5f60-4cd5-c7b6-b5db44763607"
   },
   "outputs": [],
   "source": [
    "! gcloud ai-platform predict \\\n",
    "  --region $REGION \\\n",
    "  --model $MODEL_NAME \\\n",
    "  --version $MODEL_VERSION \\\n",
    "  --json-request prediction_input.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kC1zIQ4fOKy8"
   },
   "source": [
    "### Online predictions on the whole dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yB8S7sPmOKy9"
   },
   "source": [
    "Let's now get predictions for the whole dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FvovuObyOKy9"
   },
   "outputs": [],
   "source": [
    "# Helper copied from the AI Platform console\n",
    "import googleapiclient.discovery\n",
    "from google.api_core.client_options import ClientOptions\n",
    "\n",
    "def predict_json(project, model, instances, version=None):\n",
    "    \"\"\"Send json data to a deployed model for prediction.\n",
    "\n",
    "    Args:\n",
    "        project (str): project where the Cloud ML Engine Model is deployed.\n",
    "        model (str): model name.\n",
    "        instances ([Mapping[str: Any]]): Keys should be the names of Tensors\n",
    "            your deployed model expects as inputs. Values should be datatypes\n",
    "            convertible to Tensors, or (potentially nested) lists of datatypes\n",
    "            convertible to tensors.\n",
    "        version: str, version of the model to target.\n",
    "    Returns:\n",
    "        Mapping[str: any]: dictionary of prediction results defined by the\n",
    "            model.\n",
    "    \"\"\"\n",
    "    endpoint = 'https://europe-west1-ml.googleapis.com'\n",
    "    client_options = ClientOptions(api_endpoint=endpoint)\n",
    "    service = googleapiclient.discovery.build('ml', 'v1', client_options=client_options)\n",
    "    name = 'projects/{}/models/{}'.format(project, model)\n",
    "\n",
    "    if version is not None:\n",
    "        name += '/versions/{}'.format(version)\n",
    "\n",
    "    response = service.projects().predict(\n",
    "        name=name,\n",
    "        body={'instances': instances}\n",
    "    ).execute()\n",
    "\n",
    "    if 'error' in response:\n",
    "        raise RuntimeError(response['error'])\n",
    "\n",
    "    return response['predictions']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QdSzdt9jOKy9"
   },
   "source": [
    "We need to provide our Project ID to the online predictions service.  \n",
    "Project ID is a unique identifier for the Google Cloud environment you are currently using.  \n",
    "Let's use ```gcloud config get-value project``` to get this property from gcloud:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fziJ_ByeOKy9"
   },
   "outputs": [],
   "source": [
    "PROJECT_ID = !gcloud config get-value project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tv_jBHDaOKy9"
   },
   "outputs": [],
   "source": [
    "PROJECT_ID = PROJECT_ID.get_nlstr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tErkOQP5OKy9"
   },
   "outputs": [],
   "source": [
    "from functools import partial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vmwjegyNOKy-",
    "outputId": "b7aacd98-f48f-4466-c468-23efe4e6e319"
   },
   "outputs": [],
   "source": [
    "!echo $MODEL_NAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Bai2B7nyOKy-"
   },
   "outputs": [],
   "source": [
    "get_predictions = partial(\n",
    "    predict_json,\n",
    "    project=PROJECT_ID, \n",
    "    model='kers', \n",
    "    version='v1'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Pc-xEV1kOKy-"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zm44SpgROKy-",
    "outputId": "03cf4fc8-4a03-453c-f34a-593e2cea044a"
   },
   "outputs": [],
   "source": [
    "num_batches = len(test_df)//BATCH_SIZE; num_batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5hUCgzaROKy_",
    "outputId": "19417a91-8a07-4e89-86d8-616ddda3cb71"
   },
   "outputs": [],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Pkj-CPsYOKy_",
    "outputId": "e9c2cd7c-0cdc-42a7-85ed-50e0c08a6f16"
   },
   "outputs": [],
   "source": [
    "prediction_scores = []\n",
    "df = test_df.drop('ELAPSEDTIME',axis=1)\n",
    "\n",
    "for i in tqdm(range(num_batches+1), total=num_batches, position=0):\n",
    "    batch_df = df.iloc[i*BATCH_SIZE:(i+1)*BATCH_SIZE,:]\n",
    "    pred = get_predictions(instances=batch_df.to_dict('records'))\n",
    "    #print(pred)\n",
    "    #pred = [p['dense_1'][0] for p in pred]\n",
    "    prediction_scores.extend(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3btxaJHEOKy_",
    "outputId": "4944f891-194c-485d-9d47-74851fd0cbfa"
   },
   "outputs": [],
   "source": [
    "prediction_scores[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xQelrSBbOKzB"
   },
   "source": [
    "# Clean up"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JK4546IvOKzB"
   },
   "source": [
    "Delete all versions and all models:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nu-Oe6SpOKzB"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import googleapiclient.discovery\n",
    "\n",
    "service = googleapiclient.discovery.build('ml', 'v1')\n",
    "\n",
    "project = !gcloud config get-value project\n",
    "project = project.get_nlstr()\n",
    "\n",
    "def get_models(project):\n",
    "    response = service.projects().models().list(\n",
    "        parent = 'projects/{}'.format(project)\n",
    "    ).execute()\n",
    "    \n",
    "    return response[\"models\"]\n",
    "\n",
    "def get_versions(model):\n",
    "    response = service.projects().models().versions().list(\n",
    "        parent=model\n",
    "    ).execute()\n",
    "    \n",
    "    return response[\"versions\"]\n",
    "\n",
    "def delete_version(version):\n",
    "    print(\"Deleting version: \", version[\"name\"])\n",
    "    \n",
    "    response = service.projects().models().versions().delete(\n",
    "        name=version[\"name\"]\n",
    "    ).execute()\n",
    "    \n",
    "    if \"error\" in response:\n",
    "        print(error)\n",
    "    \n",
    "    return response[\"name\"]\n",
    "\n",
    "def delete_model(model):\n",
    "    print(\"Deleting model: \", model[\"name\"])\n",
    "    \n",
    "    response = service.projects().models().delete(\n",
    "        name=model[\"name\"]\n",
    "    ).execute()\n",
    "    \n",
    "    if \"error\" in response:\n",
    "        print(error)\n",
    "\n",
    "def is_version_deleted(operation):\n",
    "    print(\"Checking status for operation: \", operation)\n",
    "    \n",
    "    response = service.projects().operations().get(\n",
    "        name=operation\n",
    "    ).execute()\n",
    "    \n",
    "    print(response)\n",
    "    if \"done\" in response:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "models = get_models(project)\n",
    "\n",
    "default_version_deletions = []\n",
    "for model in models:\n",
    "    print('Model: ', model[\"name\"])\n",
    "    versions = get_versions(model[\"name\"])\n",
    "    deletions_in_progress = []\n",
    "    for version in versions:\n",
    "        # Delete non default versions\n",
    "        if \"isDefault\" not in version:\n",
    "            versions.remove(version)\n",
    "            deletions_in_progress.append(delete_version(version))\n",
    "    while len(deletions_in_progress) > 0:\n",
    "        # Try again in 5s\n",
    "        print(\"Waiting 5s\")\n",
    "        time.sleep(5)\n",
    "        for deletion_in_progress in deletions_in_progress:\n",
    "            if is_version_deleted(deletion_in_progress):\n",
    "                print(\"Deletion completed: \", deletion_in_progress)\n",
    "                deletions_in_progress.remove(deletion_in_progress)\n",
    "    # When all default versions are deleted, remove the default version\n",
    "    default_version_deletions.append(delete_version(versions[0]))\n",
    "while len(default_version_deletions) > 0:\n",
    "    # Try again in 5s\n",
    "    print(\"Waiting 5s\")\n",
    "    time.sleep(5)\n",
    "    for default_version_deletion in default_version_deletions:\n",
    "        if is_version_deleted(default_version_deletion):\n",
    "            print(\"Deletion completed: \", default_version_deletion)\n",
    "            default_version_deletions.remove(default_version_deletion)\n",
    "# All versions deleted, now delete the model\n",
    "for model in models:\n",
    "    delete_model(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5OZw2ZwKOKzB"
   },
   "source": [
    "Delete the bucket:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "h2SpN5O_OKzB"
   },
   "outputs": [],
   "source": [
    "# Delete your bucket\n",
    "!gsutil rm -r gs://{BUCKET_NAME}"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "sample-notebook.ipynb",
   "provenance": []
  },
  "environment": {
   "name": "tf2-cpu.2-3.mnightly-2021-01-20-debian-10-test",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-cpu.2-3:mnightly-2021-01-20-debian-10-test"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}