3LayerLSTMwithConfusionMatrix.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import copy\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.dataset import Dataset\n",
    "# from Downloads import LSTMSudokuClassifier as LSTMC\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from random import randint\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "tqdm_notebook.monitor_interval = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LSTMClassifier(nn.Module):\n",
    "\n",
    "    def __init__(self, input_dim, hidden_dim, n_label, batch_size, n_layers, use_gpu):\n",
    "        super(LSTMClassifier, self).__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.batch_size = batch_size\n",
    "        self.use_gpu = use_gpu\n",
    "        self.n_layers = n_layers\n",
    "               \n",
    "        self.lstm1 = nn.LSTM(input_dim, hidden_dim, n_layers, bidirectional=True)\n",
    "        self.lstm2 = nn.LSTM(input_dim, hidden_dim, n_layers, bidirectional=True)\n",
    "        self.lstm3 = nn.LSTM(input_dim, hidden_dim, n_layers, bidirectional=True)\n",
    "        \n",
    "        self.hidden2label = nn.Sequential(torch.nn.Linear(2*hidden_dim, n_label),\n",
    "                                          torch.nn.Softmax(dim = -1))\n",
    "        self.r_hidden = self.init_hidden()\n",
    "        self.c_hidden = self.init_hidden()\n",
    "        self.s_hidden = self.init_hidden()\n",
    "\n",
    "    def init_hidden(self):\n",
    "        if self.use_gpu:\n",
    "            h0 = Variable(torch.zeros(2*n_layers, self.batch_size, self.hidden_dim).cuda())\n",
    "            c0 = Variable(torch.zeros(2*n_layers, self.batch_size, self.hidden_dim).cuda())\n",
    "        else:\n",
    "            h0 = Variable(torch.zeros(2*n_layers, self.batch_size, self.hidden_dim))\n",
    "            c0 = Variable(torch.zeros(2*n_layers, self.batch_size, self.hidden_dim))\n",
    "        return (h0, c0)\n",
    "    \n",
    "    def ManipulateSquare(self, lstm_out_square):\n",
    "        counter = 0\n",
    "        inner_stack = None\n",
    "        outer_stack = None\n",
    "        for i in range(3):\n",
    "            for j in range(3):\n",
    "                x = lstm_out_square[counter,:,:]\n",
    "                counter += 1\n",
    "                x = x.view(3,3,2*hidden_dim)#change to 2*hidden_dim\n",
    "                if inner_stack is None:\n",
    "                    inner_stack = x\n",
    "                else:\n",
    "                    inner_stack = torch.cat((inner_stack,x),1)        \n",
    "            if outer_stack is None:\n",
    "#                 print(inner_stack.shape)\n",
    "                outer_stack = inner_stack\n",
    "                inner_stack = None\n",
    "#                 print('B = ',outer_stack)\n",
    "            else:\n",
    "#                 print(inner_stack.shape)\n",
    "                outer_stack = torch.cat((outer_stack,inner_stack),0)\n",
    "#                 print('B = ',outer_stack)\n",
    "                inner_stack = None\n",
    "        return outer_stack\n",
    "\n",
    "    def forward(self, row_tensor, col_tensor, square_tensor):\n",
    "        \n",
    "        lstm_out_row, hidden_row = self.lstm1(row_tensor, self.r_hidden)\n",
    "        lstm_out_col, hidden_col = self.lstm2(col_tensor, self.c_hidden)\n",
    "        lstm_out_square, hidden_square = self.lstm3(square_tensor, self.s_hidden)\n",
    "        \n",
    "#         self.r_hidden = hidden_row\n",
    "#         self.c_hidden = hidden_col\n",
    "#         self.s_hidden = hidden_square\n",
    "        \n",
    "        # Do manipulation Here\n",
    "        #Row & Col Manipulation\n",
    "        row_col = lstm_out_row + lstm_out_col +lstm_out_square \n",
    "        #Square Manipulation\n",
    "        lstm_out = row_col \n",
    "        lstm_out_tensor = lstm_out\n",
    "#         if self.use_gpu:\n",
    "#             lstm_out_tensor = lstm_out.cuda()\n",
    "#         print(\"final tensor\", lstm_out_tensor.shape, lstm_out_tensor)\n",
    "        \n",
    "        #Sum Square Row Col\n",
    "        y  = self.hidden2label(lstm_out_tensor)\n",
    "#         print('Output Shape ', y.shape)\n",
    "        return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class InputGrid():\n",
    "    \n",
    "    def __init__(self, Grid):\n",
    "        self.Grid = Grid\n",
    "        self.gridLength = len(Grid[0])\n",
    "#         self.labels = labels\n",
    "        \n",
    "    def getlims(self, i):\n",
    "        if 0 <= i <= 2:\n",
    "            rowlims = [0,3]\n",
    "        elif 3 <= i <= 5:\n",
    "            rowlims = [3,6]\n",
    "        elif 6 <= i <= 8:\n",
    "                rowlims = [6,9]\n",
    "        return rowlims\n",
    "\n",
    "    def get_square(self, i):\n",
    "        grid_square = None\n",
    "        if i < 3:\n",
    "            if i%3 == 0:\n",
    "                grid_square = self.Grid[0:3,0:3]\n",
    "            elif i%3 == 1:\n",
    "                grid_square = self.Grid[0:3,3:6]\n",
    "            elif i%3 == 2:\n",
    "                grid_square = self.Grid[0:3,6:9]\n",
    "        elif i < 6:\n",
    "            if i%3 == 0:\n",
    "                grid_square = self.Grid[3:6,0:3]\n",
    "            elif i%3 == 1:\n",
    "                grid_square = self.Grid[3:6,3:6]\n",
    "            elif i%3 == 2:\n",
    "                grid_square = self.Grid[3:6,6:9]\n",
    "        elif i < 9:\n",
    "            if i%3 == 0:\n",
    "                grid_square = self.Grid[6:9,0:3]\n",
    "            elif i%3 == 1:\n",
    "                grid_square = self.Grid[6:9,3:6]\n",
    "            elif i%3 == 2:\n",
    "                grid_square = self.Grid[6:9,6:9]\n",
    "        return grid_square.flatten()\n",
    "                \n",
    "    \n",
    "    # takes 1D returns 2D\n",
    "    def one_hot(self, vec):\n",
    "        one_hot_matrix = []\n",
    "        for val in vec:\n",
    "            hot_vec = [0 for _ in range(9)]\n",
    "            if val > 0:\n",
    "                hot_vec[int(val)-1] = 1\n",
    "            one_hot_matrix.append(hot_vec)\n",
    "        real_one_hot = np.array(one_hot_matrix)\n",
    "        return real_one_hot\n",
    "    \n",
    "    #takes 2D returns 3D\n",
    "    def getInput(self):\n",
    "#         print(self.Grid)\n",
    "        Rows = np.zeros((9,9,9))\n",
    "        Columns = np.zeros((9,9,9))\n",
    "        Squares = np.zeros((9,9,9))\n",
    "        Labels = np.zeros((9,9,9))\n",
    "        for i in range(self.gridLength):\n",
    "            hot_row = self.one_hot(self.Grid[i,:])\n",
    "            hot_column = self.one_hot(self.Grid[:,i])\n",
    "            hot_square = self.one_hot(self.get_square(i))\n",
    "#             hot_label = self.one_hot(self.labels[i,:])\n",
    "            \n",
    "            Rows[i] = hot_row\n",
    "            Columns[i] = hot_column\n",
    "            Squares[i] = hot_square\n",
    "#             Labels[i] = hot_label\n",
    "        \n",
    "        Rows = np.reshape (Rows , (1, 81, 9))\n",
    "        Columns = np.reshape (Columns, (1, 81, 9))\n",
    "        Squares = np.reshape (Squares, (1, 81, 9))\n",
    "        Rows = np.transpose (Rows, ((1, 0, 2)))\n",
    "        Columns = np.transpose (Columns, ((1, 0, 2)))\n",
    "        Squares = np.transpose (Squares, ((1, 0, 2)))\n",
    "\n",
    "        row_tensor = Variable(torch.FloatTensor(Rows))\n",
    "        col_tensor = Variable(torch.FloatTensor(Columns))\n",
    "        square_tensor = Variable(torch.FloatTensor(Squares))\n",
    "#         label_tensor = Variable(torch.LongTensor(Labels))\n",
    "#         print(row_tensor)\n",
    "#         print(col_tensor)\n",
    "#         print(square_tensor)\n",
    "#         print(label_tensor)\n",
    "                 \n",
    "        return row_tensor, col_tensor, square_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# use_plot = True\n",
    "# use_save = True\n",
    "# if use_save:\n",
    "#     import pickle\n",
    "#     from datetime import datetime\n",
    "\n",
    "DATA_DIR = 'data'\n",
    "TRAIN_FILE = 'kaggle_sudoku.csv'\n",
    "TEST_FILE = 'test.csv'\n",
    "TRAIN_LABEL = 'train_label.txt'\n",
    "TEST_LABEL = 'test_label.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SudokuDataset(Dataset):\n",
    "    def __init__(self, fpath):\n",
    "        \n",
    "        print(fpath)\n",
    "        lines = open(fpath, 'r').read().splitlines()[1:]\n",
    "        nsamples = len(lines)\n",
    "\n",
    "        X = np.zeros((nsamples, 9*9), np.float32)  \n",
    "        Y = np.zeros((nsamples, 9*9), np.int32) \n",
    "\n",
    "        for i, line in enumerate(lines):\n",
    "            quiz, solution = line.split(\",\")\n",
    "            for j, (q, s) in enumerate(zip(quiz, solution)):\n",
    "                X[i, j], Y[i, j] = q, s\n",
    "            if i>=nsamples-1:\n",
    "                break\n",
    "\n",
    "        X = np.reshape(X, (-1, 9, 9))\n",
    "        Y = np.reshape(Y, (-1, 9, 9))\n",
    "        Y -= 1\n",
    "        self.X = X\n",
    "        self.Y = Y\n",
    "#         print(Y)\n",
    "#         print(Y.shape)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        quiz = self.X[index]\n",
    "        sol = self.Y[index]\n",
    "        return quiz, sol\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__=='__main__':\n",
    "    \n",
    "    ## parameter setting\n",
    "    epochs = 100\n",
    "    batch_size = 1\n",
    "    use_gpu = torch.cuda.is_available()\n",
    "    learning_rate = 0.001\n",
    "\n",
    "    input_dim = 9\n",
    "    hidden_dim = 200\n",
    "    n_label = 9\n",
    "    n_layers = 3\n",
    "    \n",
    "    train_path = os.path.join(DATA_DIR, TRAIN_FILE)\n",
    "    test_path = os.path.join(DATA_DIR, TEST_FILE)\n",
    "    save_filepath = 'rnn-3-100-1000.pt'\n",
    "\n",
    "\n",
    "     ### ********************create model**************************\n",
    "    model = LSTMClassifier(input_dim, hidden_dim, n_label, batch_size, n_layers, use_gpu)\n",
    "    if use_gpu:\n",
    "        model = model.cuda()\n",
    "    \n",
    "    training_set = SudokuDataset(train_path)\n",
    "    train_loader = DataLoader(training_set,\n",
    "                          batch_size,\n",
    "                          shuffle=True,\n",
    "                          num_workers=0\n",
    "                          )\n",
    "    \n",
    "#     print('TRAINING')\n",
    "\n",
    "    optimizer = optim.Adam(model.parameters())\n",
    "    loss_function = nn.CrossEntropyLoss()\n",
    "    train_loss_ = []\n",
    "    test_loss_ = []\n",
    "    train_acc_ = []\n",
    "    test_acc_ = []\n",
    "    \n",
    "    \n",
    "    def adjust_learning_rate(optimizer, epoch):\n",
    "        lr = learning_rate * (0.1 ** (epoch // 10))\n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = lr\n",
    "        return optimizer\n",
    "    \n",
    "    import pdb\n",
    "    nsamples = 1000//batch_size\n",
    "\n",
    "### training procedure\n",
    "\n",
    "#     t = trange(100, desc='Bar desc', leave=True)\n",
    "    for epoch in range(epochs):\n",
    "#         optimizer = adjust_learning_rate(optimizer, epoch)\n",
    "        total_acc = 0.0\n",
    "        total_loss = 0.0\n",
    "        total = 0.0\n",
    "\n",
    "#         model.hidden = model.init_hidden()\n",
    "\n",
    "#         t = trange(100000, desc='RNN', leave=True)\n",
    "        for i, traindata in zip(tqdm_notebook(range(nsamples)), train_loader):\n",
    "#             traindata = train_loader[i]\n",
    "            model.zero_grad()\n",
    "            train_inputs_raw, train_labels_raw = traindata\n",
    "#             print(train_inputs_raw.shape)\n",
    "\n",
    "            train_data_numpy = Variable(train_inputs_raw).data.numpy()\n",
    "            train_labels_numpy = Variable(train_labels_raw).data.numpy()\n",
    "            train_labels = Variable(torch.LongTensor(train_labels_numpy[0]))\n",
    "\n",
    "#             pdb.set_trace ()\n",
    "            input_grid = InputGrid(train_data_numpy[0])\n",
    "            row_tensor, col_tensor, square_tensor = input_grid.getInput()\n",
    "\n",
    "            if use_gpu:\n",
    "                row_tensor, col_tensor, square_tensor, train_labels = \\\n",
    "                        row_tensor.cuda(), col_tensor.cuda(), square_tensor.cuda(), train_labels.cuda()\n",
    "\n",
    "            model.hidden = model.init_hidden()\n",
    "            model.batch_size =  batch_size \n",
    "            output = model(row_tensor, col_tensor, square_tensor)\n",
    "            output = output.view(81,9)\n",
    "\n",
    "\n",
    "#             for i in range(output.size()[1]):\n",
    "            loss = loss_function(output, train_labels.view(81,))\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # calc training acc\n",
    "            _, predicted = torch.max(output.data, 1)\n",
    "#             print(predicted.shape)\n",
    "#             print('TrainPred =  ',predicted, 'While TrainLabel = ' ,train_labels.data)\n",
    "            acc = (predicted == train_labels.data.view(81)).sum()/81\n",
    "#             print(acc)\n",
    "            total += 81\n",
    "#             print(loss.data[0])\n",
    "#             print (acc)\n",
    "\n",
    "#        print(\"loss:\", loss.data[0], \"acc\", total_acc)\n",
    "            train_loss_.append(loss.data[0])\n",
    "            train_acc_.append(acc)\n",
    "\n",
    "#             t.set_description(\"Bar desc (file %i)\" % iter)\n",
    "#             tqdm.refresh(self=) # t/o show immediately the update\n",
    "\n",
    "        print('[Epoch: %3d/%3d] Training Loss: %.3f, Training Acc: %.3f' \n",
    "                  % (epoch, epochs, np.mean(train_loss_), np.mean (train_acc_)))\n",
    "    torch.save(model.state_dict(), save_filepath+'_'+str(epoch))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "### TEST\n",
    "epochs = 100\n",
    "batch_size = 1\n",
    "use_gpu = torch.cuda.is_available()\n",
    "learning_rate = 0.001\n",
    "\n",
    "input_dim = 9\n",
    "hidden_dim = 200\n",
    "n_label = 9\n",
    "n_layers = 3\n",
    "\n",
    "train_path = os.path.join(DATA_DIR, TRAIN_FILE)\n",
    "test_path = os.path.join(DATA_DIR, TEST_FILE)\n",
    "save_filepath = 'rnn-3-100-1000.pt_99'\n",
    "\n",
    "\n",
    " ### ********************create model**************************\n",
    "model = LSTMClassifier(input_dim, hidden_dim, n_label, batch_size, n_layers, use_gpu)\n",
    "if use_gpu:\n",
    "    model = model.cuda()    \n",
    "\n",
    "\n",
    "model.load_state_dict(torch.load(save_filepath))\n",
    "model.eval()\n",
    "\n",
    "# original saved file with DataParallel\n",
    "# state_dict = torch.load(save_filepath)\n",
    "# # create new OrderedDict that does not contain `module.`\n",
    "# from collections import OrderedDict\n",
    "# new_state_dict = OrderedDict()\n",
    "# for k, v in state_dict.items():\n",
    "#     print(k,v)\n",
    "#     name = k[7:] # remove `module.`\n",
    "#     new_state_dict[name] = v\n",
    "# # load params\n",
    "# model.load_state_dict(new_state_dict)\n",
    "# model.eval()\n",
    "\n",
    "total_acc = 0.0\n",
    "total = 0.0\n",
    "\n",
    "test_set = SudokuDataset(test_path)\n",
    "test_loader = DataLoader(test_set,\n",
    "                        batch_size,\n",
    "                        shuffle=True,\n",
    "                        num_workers=0\n",
    "                        )\n",
    "nsamples = 30\n",
    "test_loss_ = []\n",
    "test_acc_ = []\n",
    "predictedStore = []\n",
    "solutionStore = []\n",
    "for i, testdata in zip(tqdm_notebook(range(nsamples)), test_loader):\n",
    "    model.zero_grad()\n",
    "    train_inputs_raw, train_labels_raw = testdata\n",
    "#             print(train_inputs_raw.shape)\n",
    "\n",
    "    train_data_numpy = Variable(train_inputs_raw).data.numpy()\n",
    "    train_labels_numpy = Variable(train_labels_raw).data.numpy()\n",
    "    train_labels = Variable(torch.LongTensor(train_labels_numpy[0]))\n",
    "\n",
    "    input_grid = InputGrid(train_data_numpy[0])\n",
    "    row_tensor, col_tensor, square_tensor = input_grid.getInput()\n",
    "\n",
    "    if use_gpu:\n",
    "        row_tensor, col_tensor, square_tensor, train_labels = \\\n",
    "                row_tensor.cuda(), col_tensor.cuda(), square_tensor.cuda(), train_labels.cuda()\n",
    "\n",
    "    model.hidden = model.init_hidden()\n",
    "    model.batch_size = batch_size \n",
    "    output = model(row_tensor, col_tensor, square_tensor)\n",
    "    output = output.view(81,9)\n",
    "\n",
    "#             for i in range(output.size()[1]):\n",
    "#         loss = loss_function(output, train_labels.view(81,))\n",
    "# #         loss.backward()\n",
    "# #         optimizer.step()\n",
    "\n",
    "    # calc training acc\n",
    "    _, predicted = torch.max(output.data, 1)\n",
    "    predictedStore.append(predicted.view(81,).cpu().numpy().tolist())\n",
    "    solutionStore.append(train_labels.data.view(81,).cpu().numpy().tolist())\n",
    "#     print('TrainPred = ', predicted.view(9,9), 'While TrainLabel = ' ,train_labels.data.view(9,9))\n",
    "    acc = (predicted == train_labels.data.view(81)).sum()/81\n",
    "#             print(acc)\n",
    "#             print(loss.data[0])\n",
    "#             print (acc)\n",
    "\n",
    "#        print(\"loss:\", loss.data[0], \"acc\", total_acc)\n",
    "#     test_loss_.append(loss.data[0])\n",
    "    test_acc_.append(acc)\n",
    "\n",
    "#             t.set_description(\"Bar desc (file %i)\" % iter)\n",
    "#             tqdm.refresh(self=) # t/o show immediately the update\n",
    "\n",
    "predictedStore = [i for arr in predictedStore for i in arr]\n",
    "solutionStore = [i for arr in solutionStore for i in arr]\n",
    "\n",
    "print('[Epoch: %3d/%3d], Testing Acc: %.3f' % (1, 1, np.mean (test_acc_)))\n",
    "\n",
    "np.savetxt(\"predictedStore.csv\", predictedStore, delimiter=\",\", fmt='%s')\n",
    "np.savetxt(\"solutionStore.csv\", solutionStore, delimiter=\",\", fmt='%s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "y_pred = np.array(predictedStore) \n",
    "y_test = np.array(solutionStore)\n",
    "cnf_matrix = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "\n",
    "class_names = [str(i+1) for i in range(9)]\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, classes=class_names,\n",
    "                      title='Confusion matrix, without normalization')\n",
    "\n",
    "# Plot normalized confusion matrix\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,\n",
    "                      title='Normalized confusion matrix')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# f1 = np.mean(2*[(precision*recall)/(precision+recall)])\n",
    "\n",
    "F1 = []\n",
    "for i in range(len(cnf_matrix)):\n",
    "    tp = 0\n",
    "    fp = 0\n",
    "    fn = 0\n",
    "    row = cnf_matrix[i,:]\n",
    "    col = cnf_matrix[:,i]\n",
    "    tp = cnf_matrix[i,i]\n",
    "#     print(tp)\n",
    "    fp = np.sum(col) - tp\n",
    "#     print(fp)\n",
    "    fn = np.sum(row) - tp\n",
    "#     print(fn)\n",
    "    precision = tp/(tp+fp)\n",
    "    recall = tp/(tp+fn)\n",
    "    f1 = 2*(precision*recall)/(precision+recall)\n",
    "    F1.append(f1)\n",
    "print(np.mean(F1))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}