notebooks/multi_convnet_sentiment_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi-ConvNet Sentiment Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we concatenate the outputs of *multiple, parallel convolutional layers* to classify IMDB movie reviews by their sentiment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow\n",
    "from tensorflow.keras.datasets import imdb\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Model # new!\n",
    "from tensorflow.keras.layers import Input, concatenate # new! \n",
    "from tensorflow.keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, Conv1D, GlobalMaxPooling1D\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "from sklearn.metrics import roc_auc_score \n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# output directory name:\n",
    "output_dir = 'model_output/multiconv'\n",
    "\n",
    "# training:\n",
    "epochs = 4\n",
    "batch_size = 128\n",
    "\n",
    "# vector-space embedding: \n",
    "n_dim = 64\n",
    "n_unique_words = 5000 \n",
    "max_review_length = 400\n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2 \n",
    "\n",
    "# convolutional layer architecture:\n",
    "n_conv_1 = n_conv_2 = n_conv_3 = 256 \n",
    "k_conv_1 = 3\n",
    "k_conv_2 = 2\n",
    "k_conv_3 = 4\n",
    "\n",
    "# dense layer architecture: \n",
    "n_dense = 256\n",
    "dropout = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
    "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_layer = Input(shape=(max_review_length,), \n",
    "                    dtype='int16', name='input') \n",
    "\n",
    "# embedding: \n",
    "embedding_layer = Embedding(n_unique_words, n_dim, \n",
    "                            name='embedding')(input_layer)\n",
    "drop_embed_layer = SpatialDropout1D(drop_embed, \n",
    "                                    name='drop_embed')(embedding_layer)\n",
    "\n",
    "# three parallel convolutional streams: \n",
    "conv_1 = Conv1D(n_conv_1, k_conv_1, \n",
    "                activation='relu', name='conv_1')(drop_embed_layer)\n",
    "maxp_1 = GlobalMaxPooling1D(name='maxp_1')(conv_1)\n",
    "\n",
    "conv_2 = Conv1D(n_conv_2, k_conv_2, \n",
    "                activation='relu', name='conv_2')(drop_embed_layer)\n",
    "maxp_2 = GlobalMaxPooling1D(name='maxp_2')(conv_2)\n",
    "\n",
    "conv_3 = Conv1D(n_conv_3, k_conv_3, \n",
    "                activation='relu', name='conv_3')(drop_embed_layer)\n",
    "maxp_3 = GlobalMaxPooling1D(name='maxp_3')(conv_3)\n",
    "\n",
    "# concatenate the activations from the three streams: \n",
    "concat = concatenate([maxp_1, maxp_2, maxp_3])\n",
    "\n",
    "# dense hidden layers: \n",
    "dense_layer = Dense(n_dense, \n",
    "                    activation='relu', name='dense')(concat)\n",
    "drop_dense_layer = Dropout(dropout, name='drop_dense')(dense_layer)\n",
    "dense_2 = Dense(int(n_dense/4), \n",
    "                activation='relu', name='dense_2')(drop_dense_layer)\n",
    "dropout_2 = Dropout(dropout, name='drop_dense_2')(dense_2)\n",
    "\n",
    "# sigmoid output layer: \n",
    "predictions = Dense(1, activation='sigmoid', name='output')(dropout_2)\n",
    "\n",
    "# create model: \n",
    "model = Model(input_layer, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input (InputLayer)              (None, 400)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 400, 64)      320000      input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "drop_embed (SpatialDropout1D)   (None, 400, 64)      0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_1 (Conv1D)                 (None, 398, 256)     49408       drop_embed[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_2 (Conv1D)                 (None, 399, 256)     33024       drop_embed[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_3 (Conv1D)                 (None, 397, 256)     65792       drop_embed[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "maxp_1 (GlobalMaxPooling1D)     (None, 256)          0           conv_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "maxp_2 (GlobalMaxPooling1D)     (None, 256)          0           conv_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "maxp_3 (GlobalMaxPooling1D)     (None, 256)          0           conv_3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 768)          0           maxp_1[0][0]                     \n",
      "                                                                 maxp_2[0][0]                     \n",
      "                                                                 maxp_3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 256)          196864      concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "drop_dense (Dropout)            (None, 256)          0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 64)           16448       drop_dense[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "drop_dense_2 (Dropout)          (None, 64)           0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 1)            65          drop_dense_2[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 681,601\n",
      "Trainable params: 681,601\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/4\n",
      "25000/25000 [==============================] - 107s 4ms/step - loss: 0.4853 - acc: 0.7374 - val_loss: 0.2925 - val_acc: 0.8782\n",
      "Epoch 2/4\n",
      "25000/25000 [==============================] - 106s 4ms/step - loss: 0.2460 - acc: 0.9019 - val_loss: 0.2620 - val_acc: 0.8941\n",
      "Epoch 3/4\n",
      "25000/25000 [==============================] - 106s 4ms/step - loss: 0.1719 - acc: 0.9381 - val_loss: 0.2776 - val_acc: 0.8882\n",
      "Epoch 4/4\n",
      "25000/25000 [==============================] - 106s 4ms/step - loss: 0.1230 - acc: 0.9570 - val_loss: 0.2922 - val_acc: 0.8943\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f5003803588>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+\"/weights.02.hdf5\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = model.predict(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEPRJREFUeJzt3X2snnV9x/H3Ryo+I2CLYS1bIVYnkiyyButMnLOGR0P5A5aaOSpp1sQx55zZhtsfXUAS3BOORHGddBbjBMbMaARHOh7itghyEIc8jNABgw4mRwvVjfhQ/e6P+1d24Hfa3j336bl72vcrObmv63f9ruv6/npO+znXY1NVSJI01UvGXYAk6cBjOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKmzYNwFzNTChQtr6dKl4y5DeqHvPTT4POJN461Dmsbdd9/9napaNEzfeRsOS5cuZWJiYtxlSC/0T+8afL7n9nFWIU0ryX8O29fTSpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkzrx9QlqSxmnpRTeOZb+PXXbWnOxnr0cOSTYmeTrJfVPajk6yJcnD7fOo1p4kVyTZmuTeJCdPWWdN6/9wkjVT2n8xybfaOlckyWwPUpK0b4Y5rfQ54PQXtV0E3FJVy4Bb2jzAGcCy9rUOuBIGYQKsB94GnAKs3xUorc+6Keu9eF+SpDm213Coqq8C21/UvArY1KY3AedMab+6Bu4AjkxyLHAasKWqtlfVM8AW4PS27Iiq+lpVFXD1lG1JksZkphekX19VTwG0z2Na+2LgiSn9trW2PbVvm6ZdkjRGs3230nTXC2oG7dNvPFmXZCLJxOTk5AxLlCTtzUzD4dvtlBDt8+nWvg04bkq/JcCTe2lfMk37tKpqQ1Utr6rlixYN9f9VSJJmYKbhsBnYdcfRGuCGKe3nt7uWVgA72mmnm4FTkxzVLkSfCtzcln0/yYp2l9L5U7YlSRqTvT7nkOSLwLuAhUm2Mbjr6DLguiRrgceB81r3m4Azga3Ac8AFAFW1PcklwF2t38VVtesi9wcZ3BH1CuAr7UuSNEZ7DYeqet9uFq2cpm8BF+5mOxuBjdO0TwAn7a0OSdLc8fUZkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqTOgnEXMA5LL7pxLPt97LKzxrJfSdpXIx05JPlIkvuT3Jfki0lenuT4JHcmeTjJtUkOb31f1ua3tuVLp2znY639oSSnjTYkSdKoZhwOSRYDvw0sr6qTgMOA1cAngMurahnwDLC2rbIWeKaq3gBc3vqR5MS23luA04FPJzlspnVJkkY36jWHBcArkiwAXgk8BbwbuL4t3wSc06ZXtXna8pVJ0tqvqaofVtWjwFbglBHrkiSNYMbhUFX/BfwZ8DiDUNgB3A08W1U7W7dtwOI2vRh4oq27s/V/3dT2adZ5gSTrkkwkmZicnJxp6ZKkvRjltNJRDH7rPx74GeBVwBnTdK1dq+xm2e7a+8aqDVW1vKqWL1q0aN+LliQNZZTTSu8BHq2qyar6MfAl4JeAI9tpJoAlwJNtehtwHEBb/lpg+9T2adaRJI3BKOHwOLAiySvbtYOVwAPAbcC5rc8a4IY2vbnN05bfWlXV2le3u5mOB5YBXx+hLknSiGb8nENV3ZnkeuAbwE7gHmADcCNwTZKPt7ar2ipXAZ9PspXBEcPqtp37k1zHIFh2AhdW1U9mWpckaXQjPQRXVeuB9S9qfoRp7jaqqh8A5+1mO5cCl45SiyRp9vj6DElSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSZ6RwSHJkkuuT/HuSB5O8PcnRSbYkebh9HtX6JskVSbYmuTfJyVO2s6b1fzjJmlEHJUkazahHDn8J/GNV/TzwC8CDwEXALVW1DLilzQOcASxrX+uAKwGSHA2sB94GnAKs3xUokqTxmHE4JDkCeCdwFUBV/aiqngVWAZtat03AOW16FXB1DdwBHJnkWOA0YEtVba+qZ4AtwOkzrUuSNLpRjhxOACaBv0lyT5LPJnkV8PqqegqgfR7T+i8Gnpiy/rbWtrt2SdKYjBIOC4CTgSur6q3A//L/p5Cmk2naag/t/QaSdUkmkkxMTk7ua72SpCGNEg7bgG1VdWebv55BWHy7nS6ifT49pf9xU9ZfAjy5h/ZOVW2oquVVtXzRokUjlC5J2pMZh0NV/TfwRJI3taaVwAPAZmDXHUdrgBva9Gbg/HbX0gpgRzvtdDNwapKj2oXoU1ubJGlMFoy4/oeALyQ5HHgEuIBB4FyXZC3wOHBe63sTcCawFXiu9aWqtie5BLir9bu4qraPWJckaQQjhUNVfRNYPs2ildP0LeDC3WxnI7BxlFokSbPHJ6QlSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSZ2RwyHJYUnuSfLlNn98kjuTPJzk2iSHt/aXtfmtbfnSKdv4WGt/KMlpo9YkSRrNbBw5fBh4cMr8J4DLq2oZ8AywtrWvBZ6pqjcAl7d+JDkRWA28BTgd+HSSw2ahLknSDI0UDkmWAGcBn23zAd4NXN+6bALOadOr2jxt+crWfxVwTVX9sKoeBbYCp4xSlyRpNKMeOXwS+H3gp23+dcCzVbWzzW8DFrfpxcATAG35jtb/+fZp1pEkjcGMwyHJe4Gnq+ruqc3TdK29LNvTOi/e57okE0kmJicn96leSdLwRjlyeAdwdpLHgGsYnE76JHBkkgWtzxLgyTa9DTgOoC1/LbB9avs067xAVW2oquVVtXzRokUjlC5J2pMZh0NVfayqllTVUgYXlG+tql8DbgPObd3WADe06c1tnrb81qqq1r663c10PLAM+PpM65IkjW7B3rvssz8ArknyceAe4KrWfhXw+SRbGRwxrAaoqvuTXAc8AOwELqyqn+yHuiRJQ5qVcKiq24Hb2/QjTHO3UVX9ADhvN+tfClw6G7VIkkbnE9KSpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqzDgckhyX5LYkDya5P8mHW/vRSbYkebh9HtXak+SKJFuT3Jvk5CnbWtP6P5xkzejDkiSNYpQjh53AR6vqzcAK4MIkJwIXAbdU1TLgljYPcAawrH2tA66EQZgA64G3AacA63cFiiRpPGYcDlX1VFV9o01/H3gQWAysAja1bpuAc9r0KuDqGrgDODLJscBpwJaq2l5VzwBbgNNnWpckaXSzcs0hyVLgrcCdwOur6ikYBAhwTOu2GHhiymrbWtvu2qfbz7okE0kmJicnZ6N0SdI0Rg6HJK8G/h74nar63p66TtNWe2jvG6s2VNXyqlq+aNGifS9WkjSUkcIhyUsZBMMXqupLrfnb7XQR7fPp1r4NOG7K6kuAJ/fQLkkak1HuVgpwFfBgVf3FlEWbgV13HK0BbpjSfn67a2kFsKOddroZODXJUe1C9KmtTZI0JgtGWPcdwK8D30ryzdb2h8BlwHVJ1gKPA+e1ZTcBZwJbgeeACwCqanuSS4C7Wr+Lq2r7CHVJkkY043Coqn9h+usFACun6V/AhbvZ1kZg40xrkSTNLp+QliR1DAdJUsdwkCR1DAdJUsdwkCR1RrmVVfto6UU3jm3fj1121tj2LWn+8chBktQxHCRJHcNBktTxmoOkeWuc1/EOdh45SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6PiF9iBjXk6S+DVaanwwHSSPzNRYHH08rSZI6hoMkqeNpJe1X/u93c8dTO5pNhoMOWuP4x/KaE77LihNeN+f7lWab4SDNsjse+S6r/S1e85zXHCRJHcNBktQxHCRJHcNBktQxHCRJnQMmHJKcnuShJFuTXDTueiTpUHZAhEOSw4BPAWcAJwLvS3LieKuSpEPXAREOwCnA1qp6pKp+BFwDrBpzTZJ0yDpQwmEx8MSU+W2tTZI0BgfKE9KZpq26Tsk6YF2b/Z8kD81wfwuB78xw3fnKMc+Btz8/9d653O0ufo8PAfnESGP+uWE7HijhsA04bsr8EuDJF3eqqg3AhlF3lmSiqpaPup35xDEf/A618YJj3p8OlNNKdwHLkhyf5HBgNbB5zDVJ0iHrgDhyqKqdSX4LuBk4DNhYVfePuSxJOmQdEOEAUFU3ATfN0e5GPjU1Dznmg9+hNl5wzPtNqrrrvpKkQ9yBcs1BknQAOWjDYW+v40jysiTXtuV3Jlk691XOriHG/LtJHkhyb5Jbkgx9W9uBatjXriQ5N0klmfd3tgwz5iS/2r7X9yf527mucbYN8bP9s0luS3JP+/k+cxx1zpYkG5M8neS+3SxPkivan8e9SU6e9SKq6qD7YnBR+z+AE4DDgX8DTnxRn98EPtOmVwPXjrvuORjzrwCvbNMfPBTG3Pq9BvgqcAewfNx1z8H3eRlwD3BUmz9m3HXPwZg3AB9s0ycCj4277hHH/E7gZOC+3Sw/E/gKg2fEVgB3znYNB+uRwzCv41gFbGrT1wMrk0z3MN58sdcxV9VtVfVcm72DwfMk89mwr125BPgT4AdzWdx+MsyYfwP4VFU9A1BVT89xjbNtmDEXcESbfi3TPCc1n1TVV4Hte+iyCri6Bu4Ajkxy7GzWcLCGwzCv43i+T1XtBHYA8/l/ht/XV5CsZfCbx3y21zEneStwXFV9eS4L24+G+T6/EXhjkn9NckeS0+esuv1jmDH/MfD+JNsY3PX4obkpbWz2+yuHDphbWWfZMK/jGOqVHfPI0ONJ8n5gOfDL+7Wi/W+PY07yEuBy4ANzVdAcGOb7vIDBqaV3MTg6/OckJ1XVs/u5tv1lmDG/D/hcVf15krcDn29j/un+L28s9vu/XwfrkcMwr+N4vk+SBQwORfd0GHegG+oVJEneA/wRcHZV/XCOattf9jbm1wAnAbcneYzBudnN8/yi9LA/2zdU1Y+r6lHgIQZhMV8NM+a1wHUAVfU14OUM3rt0sBrq7/soDtZwGOZ1HJuBNW36XODWald65qm9jrmdYvkrBsEw389Dw17GXFU7qmphVS2tqqUMrrOcXVUT4yl3Vgzzs/0PDG4+IMlCBqeZHpnTKmfXMGN+HFgJkOTNDMJhck6rnFubgfPbXUsrgB1V9dRs7uCgPK1Uu3kdR5KLgYmq2gxcxeDQcyuDI4bV46t4dEOO+U+BVwN/1669P15VZ4+t6BENOeaDypBjvhk4NckDwE+A36uq746v6tEMOeaPAn+d5CMMTq98YD7/spfkiwxOCy5s11HWAy8FqKrPMLiuciawFXgOuGDWa5jHf36SpP3kYD2tJEkageEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSer8H4OPzDiMhb+sAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f28613d4518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=0.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'96.18'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}