single_input/Multi-classification with a single Input channel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning based Pipeline with Single Input for Multi-class Patent Classification\n",
    "In this nootebook,  we used the full text of most important sections in patent  (title, abstract, technical fields, background, summary, and independent claim) as one input to the deep learning model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Recognition code, particularly for a disk-shap...\n",
       "1    Optical pickup apparatus for recording/reprodu...\n",
       "2    Large capacity data sales mediation system, se...\n",
       "3    BRIDGE FOR A CLIENT-SERVER ENVIRONMENT. A soft...\n",
       "4    PROCESSOR HAVING SECTIONS OPERATING AT DIFFERE...\n",
       "Name: TEXT, dtype: object"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"../datasets/allITPatTextWith_Metadata.csv\",  encoding = \"ISO-8859-1\", error_bad_lines=False)\n",
    "df.columns =['ID','TI','AB','TECHF','BACKG','SUMM','CLMS','ICM','AY','IPC','REF','PA','INV']\n",
    "\n",
    "df.dropna(subset=['ICM'], inplace=True)\n",
    "df.fillna(value='', inplace=True)\n",
    "\n",
    "df['TEXT'] = df['TI'] +'. '+ df['AB'] +'. '+ df['TECHF']+'. '+ df['BACKG']+'. '+ df['SUMM']+'. '+ df['CLMS']\n",
    "\n",
    "\n",
    "df.fillna(value='', inplace=True)\n",
    "\n",
    "df.TEXT.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.53 s, sys: 56.8 ms, total: 2.58 s\n",
      "Wall time: 2.59 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#preprocess of list fields\n",
    "#convert all IPCs in df into one list\n",
    "def toList(s):\n",
    "    \"\"\"\n",
    "    this method is to convert the list of IPCs in each row from a string to a python List\n",
    "    \"\"\"\n",
    "    s  = s.translate ({ord(c): \" \" for c in \"[]\"})\n",
    "    ss= []\n",
    "    for cls in s.strip().split(','):\n",
    "        ss.append(cls.strip())\n",
    "    return ss\n",
    "\n",
    "#apply toList method on all rows in the DF\n",
    "df['PA'] = df['PA'].map(lambda pa :   toList(pa))\n",
    "df['INV'] = df['INV'].map(lambda inv :   toList(inv))\n",
    "\n",
    "df.head()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.49 s, sys: 22 ms, total: 5.51 s\n",
      "Wall time: 5.52 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#also preprocess of list fields\n",
    "def metadataPreprocessing(input):\n",
    "    newInput=' '\n",
    "    for item in input:\n",
    "        item = item.translate ({ord(c): \" \" for c in \"!@#$%^&*()'[]{};:,./<>?\\|`~°=\\\"+\"})\n",
    "        itms=' '\n",
    "        for itm in item.split():\n",
    "            itms= itms +' '+itm.strip()\n",
    "        newInput = newInput + ' '+ itms.strip().replace(' ','_')\n",
    "    return newInput.strip()\n",
    "\n",
    "df['PA'] = df['PA'].map(lambda pa :   metadataPreprocessing(pa))\n",
    "df['INV'] = df['INV'].map(lambda inv :   metadataPreprocessing(inv))\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocessing \n",
    "standardStopwordFile = \"sources/stopwords/stopwords-all.txt\"\n",
    "#generalWordsFile = \"sources/Clariant/generalWords.txt\"\n",
    "\n",
    "#loading terms from a file to a set\n",
    "def get_terms_from_file(filePath):\n",
    "    terms = set(line.strip() for line in open(filePath))\n",
    "    return terms\n",
    "\n",
    "#remove undiserd terms\n",
    "def remove_terms(termSet, phrase):\n",
    "    newPhrase = \"\"\n",
    "    for term in phrase.split():\n",
    "        if term.strip() not in termSet and len(term.strip())>2:\n",
    "            newPhrase = newPhrase + \" \" + term.strip()\n",
    "\n",
    "\n",
    "\n",
    "def clean_texts(doc):\n",
    "    #Remove punctuation from texts\n",
    "    doc = doc.translate ({ord(c): ' ' for c in \"0123456789!@#$%^&*()'/[]{};:,./<>?\\|`~°=\\\"+\"})\n",
    "    # split into tokens by white space\n",
    "    tokens = doc.lower().strip().split()\n",
    "    \n",
    "    # filter out stop words\n",
    "    stop_words = get_terms_from_file(standardStopwordFile)\n",
    "    #generalStopwords = get_terms_from_file(generalWordsFile)\n",
    "\n",
    "    \n",
    "    tokens = [w.strip('-')  for w in tokens if  w not in stop_words ]\n",
    "    # filter out short and long  tokens\n",
    "    output = [word for word in tokens if len(word.strip()) > 2 and len(word) < 30 ]\n",
    "    output = \" \".join(output)\n",
    "    #apply stemming\n",
    "    #output = stem_text(output)\n",
    "    return output\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 1 µs, total: 5 µs\n",
      "Wall time: 9.54 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "apply simple preprocessing on text\n",
    "df['TI'] = df['TI'].map(lambda line : clean_texts(line))\n",
    "df['AB'] = df['AB'].map(lambda line : clean_texts(line))\n",
    "df['TECHF'] = df['TECHF'].map(lambda line : clean_texts(line))\n",
    "df['BACKG'] = df['BACKG'].map(lambda line : clean_texts(line))\n",
    "df['SUMM'] = df['SUMM'].map(lambda line : clean_texts(line))\n",
    "df['CLMS'] = df['CLMS'].map(lambda line : clean_texts(line))\n",
    "\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "581\n",
      "number of remaining documents in the dataset is:  403726\n",
      "Number of unique labels is:  42\n"
     ]
    }
   ],
   "source": [
    "#process the ICM codes and #related-patents\n",
    "df['ICM'] = df['ICM'].map(lambda icmCode : icmCode[:4])  \n",
    "\n",
    "df_ICMs = df.groupby(['ICM'])\n",
    "df_ICMs = df_ICMs.size().reset_index(name='Docs')\n",
    "\n",
    "print(len(df_ICMs.ICM.unique()))\n",
    "#filter out the rows with #docs less than N documents\n",
    "df_ICMOut =  df_ICMs[df_ICMs['Docs'] >= 500]\n",
    "\n",
    "#filter out rows of the original dataframe df accordding to df_ICMOut\n",
    "ICMList = df_ICMOut['ICM'].tolist()\n",
    "df = df[df.ICM.isin(ICMList)]\n",
    "\n",
    "icmCount = df_ICMs.count().tolist()[0]\n",
    "\n",
    "print( 'number of remaining documents in the dataset is: ',len(df))\n",
    "\n",
    "print('Number of unique labels is: ', len(df.ICM.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>TI</th>\n",
       "      <th>AB</th>\n",
       "      <th>TECHF</th>\n",
       "      <th>BACKG</th>\n",
       "      <th>SUMM</th>\n",
       "      <th>CLMS</th>\n",
       "      <th>ICM</th>\n",
       "      <th>AY</th>\n",
       "      <th>IPC</th>\n",
       "      <th>REF</th>\n",
       "      <th>PA</th>\n",
       "      <th>INV</th>\n",
       "      <th>TEXT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>365576</th>\n",
       "      <td>PCT1995008810-0</td>\n",
       "      <td>auditing</td>\n",
       "      <td>auditing interface means steps providing audit...</td>\n",
       "      <td>auditing auditing auditing enables create proc...</td>\n",
       "      <td>businesses industries legislative regulatory q...</td>\n",
       "      <td>ofauditing interface means steps providing aud...</td>\n",
       "      <td>auditing interface means steps providing audit...</td>\n",
       "      <td>G06F</td>\n",
       "      <td>1994</td>\n",
       "      <td>[G06F017-40, G06F017-60, G06Q0030-00, G09B0007...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>WRIGHT_GREGORY_ALLAN</td>\n",
       "      <td>auditing. auditing interface means steps provi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307355</th>\n",
       "      <td>PCT2011151924-0</td>\n",
       "      <td></td>\n",
       "      <td>executing predetermined processed preset key p...</td>\n",
       "      <td></td>\n",
       "      <td>predetermined target executing predetermined c...</td>\n",
       "      <td>tech mentioned ipsec enciphering encryption ke...</td>\n",
       "      <td>key advance executing predetermined timing pre...</td>\n",
       "      <td>H04L</td>\n",
       "      <td>2010</td>\n",
       "      <td>[H04L0009-16, H04L0009-08]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>MIZUMAKI_Masayoshi</td>\n",
       "      <td>. executing predetermined processed preset key...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131569</th>\n",
       "      <td>PCT2008097717-0</td>\n",
       "      <td>non-imaging light collector electro-optical sc...</td>\n",
       "      <td>extended working range electro-optical scanner...</td>\n",
       "      <td>concerns electro-optical scanner reading bar c...</td>\n",
       "      <td>laser bar reader scanner reading bar decoding ...</td>\n",
       "      <td>concerns extended range electro-optical scanne...</td>\n",
       "      <td></td>\n",
       "      <td>G06K</td>\n",
       "      <td>2008</td>\n",
       "      <td>[G06K0007-10, G06K0007-14]</td>\n",
       "      <td></td>\n",
       "      <td>SYMBOL_TECHNOLOGIES_INC_BARKAN_Edward_D_DRZYMA...</td>\n",
       "      <td>BARKAN_Edward_D_DRZYMALA_Mark</td>\n",
       "      <td>non-imaging light collector electro-optical sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55896</th>\n",
       "      <td>PCT2002080552-0</td>\n",
       "      <td>virtual personalized channel</td>\n",
       "      <td>management creates personalized channel end-us...</td>\n",
       "      <td>virtual personalized channelfield personalizin...</td>\n",
       "      <td>artphilips electronics markets personal video ...</td>\n",
       "      <td>pvr decribed lets watch live programs recorded...</td>\n",
       "      <td>management creating personalized channel end-u...</td>\n",
       "      <td>H04N</td>\n",
       "      <td>2002</td>\n",
       "      <td>[H04N007-173, G11B0027-00, H04L0012-28, H04N00...</td>\n",
       "      <td></td>\n",
       "      <td>KONINKLIJKE_PHILIPS_ELECTRONICS_N_V</td>\n",
       "      <td>VAN_EE_Jan</td>\n",
       "      <td>virtual personalized channel. management creat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59881</th>\n",
       "      <td>PCT2006070048-0</td>\n",
       "      <td>limiting traffic communications systems</td>\n",
       "      <td>limiting traffic communications based monitori...</td>\n",
       "      <td>limiting traffic communications exclusively li...</td>\n",
       "      <td>enables entities equipment nodes comprise voic...</td>\n",
       "      <td>provided limiting traffic communications monit...</td>\n",
       "      <td></td>\n",
       "      <td>G06F</td>\n",
       "      <td>2005</td>\n",
       "      <td>[G06F0011-00, H04L0012-26, H04L0029-06, H04L00...</td>\n",
       "      <td></td>\n",
       "      <td>NOKIA_CORPORATION_WANG_Hao_KAHADUWE_Ajit</td>\n",
       "      <td>WANG_Hao_KAHADUWE_Ajit</td>\n",
       "      <td>limiting traffic communications systems. limit...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     ID                                                 TI  \\\n",
       "365576  PCT1995008810-0                                           auditing   \n",
       "307355  PCT2011151924-0                                                      \n",
       "131569  PCT2008097717-0  non-imaging light collector electro-optical sc...   \n",
       "55896   PCT2002080552-0                       virtual personalized channel   \n",
       "59881   PCT2006070048-0            limiting traffic communications systems   \n",
       "\n",
       "                                                       AB  \\\n",
       "365576  auditing interface means steps providing audit...   \n",
       "307355  executing predetermined processed preset key p...   \n",
       "131569  extended working range electro-optical scanner...   \n",
       "55896   management creates personalized channel end-us...   \n",
       "59881   limiting traffic communications based monitori...   \n",
       "\n",
       "                                                    TECHF  \\\n",
       "365576  auditing auditing auditing enables create proc...   \n",
       "307355                                                      \n",
       "131569  concerns electro-optical scanner reading bar c...   \n",
       "55896   virtual personalized channelfield personalizin...   \n",
       "59881   limiting traffic communications exclusively li...   \n",
       "\n",
       "                                                    BACKG  \\\n",
       "365576  businesses industries legislative regulatory q...   \n",
       "307355  predetermined target executing predetermined c...   \n",
       "131569  laser bar reader scanner reading bar decoding ...   \n",
       "55896   artphilips electronics markets personal video ...   \n",
       "59881   enables entities equipment nodes comprise voic...   \n",
       "\n",
       "                                                     SUMM  \\\n",
       "365576  ofauditing interface means steps providing aud...   \n",
       "307355  tech mentioned ipsec enciphering encryption ke...   \n",
       "131569  concerns extended range electro-optical scanne...   \n",
       "55896   pvr decribed lets watch live programs recorded...   \n",
       "59881   provided limiting traffic communications monit...   \n",
       "\n",
       "                                                     CLMS   ICM    AY  \\\n",
       "365576  auditing interface means steps providing audit...  G06F  1994   \n",
       "307355  key advance executing predetermined timing pre...  H04L  2010   \n",
       "131569                                                     G06K  2008   \n",
       "55896   management creating personalized channel end-u...  H04N  2002   \n",
       "59881                                                      G06F  2005   \n",
       "\n",
       "                                                      IPC REF  \\\n",
       "365576  [G06F017-40, G06F017-60, G06Q0030-00, G09B0007...       \n",
       "307355                         [H04L0009-16, H04L0009-08]       \n",
       "131569                         [G06K0007-10, G06K0007-14]       \n",
       "55896   [H04N007-173, G11B0027-00, H04L0012-28, H04N00...       \n",
       "59881   [G06F0011-00, H04L0012-26, H04L0029-06, H04L00...       \n",
       "\n",
       "                                                       PA  \\\n",
       "365576                                                      \n",
       "307355                                                      \n",
       "131569  SYMBOL_TECHNOLOGIES_INC_BARKAN_Edward_D_DRZYMA...   \n",
       "55896                 KONINKLIJKE_PHILIPS_ELECTRONICS_N_V   \n",
       "59881            NOKIA_CORPORATION_WANG_Hao_KAHADUWE_Ajit   \n",
       "\n",
       "                                  INV  \\\n",
       "365576           WRIGHT_GREGORY_ALLAN   \n",
       "307355             MIZUMAKI_Masayoshi   \n",
       "131569  BARKAN_Edward_D_DRZYMALA_Mark   \n",
       "55896                      VAN_EE_Jan   \n",
       "59881          WANG_Hao_KAHADUWE_Ajit   \n",
       "\n",
       "                                                     TEXT  \n",
       "365576  auditing. auditing interface means steps provi...  \n",
       "307355  . executing predetermined processed preset key...  \n",
       "131569  non-imaging light collector electro-optical sc...  \n",
       "55896   virtual personalized channel. management creat...  \n",
       "59881   limiting traffic communications systems. limit...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#preprocess all documents\n",
    "#df['TEXT'] = df['TEXT'].map(lambda line : clean_texts(line))\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "df = shuffle(df)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(363353,)\n",
      "(40373,)\n"
     ]
    }
   ],
   "source": [
    "# lets take n% data as training and remaining m% for test.\n",
    "train_size = int(len(df) * .9)\n",
    "\n",
    "train_TI = df['TEXT'][:train_size]\n",
    "train_ICM= df['ICM'][:train_size]\n",
    "train_ID= df['ID'][:train_size]\n",
    "\n",
    "test_TI = df['TEXT'][train_size:]\n",
    "test_ICM = df['ICM'][train_size:]\n",
    "test_ID = df['ID'][train_size:]\n",
    "\n",
    "\n",
    "#metadata\n",
    "train_pa_series = df['PA'][:train_size]\n",
    "test_pa_series = df['PA'][train_size:]\n",
    "\n",
    "train_inv_series = df['INV'][:train_size]\n",
    "test_inv_series = df['INV'][train_size:]\n",
    "\n",
    "\n",
    "print(train_TI.shape)\n",
    "print(test_TI.shape)\n",
    "\n",
    "#free up some memory space\n",
    "#df.iloc[0:0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#preparing text documents and labels for deep learning\n",
    "\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import one_hot\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 97672 words in PA\n",
      "Found 280522 words in INV\n"
     ]
    }
   ],
   "source": [
    "#PA\n",
    "pa_inv_vocab_size = 2000\n",
    "pa_tokenizer = Tokenizer(num_words=pa_inv_vocab_size,  filters='!\"#$%&()*+,./:;<=>?@[\\]^`{|}~', lower=True, split=' ', char_level=False, oov_token=None)\n",
    "pa_tokenizer.fit_on_texts(train_pa_series)\n",
    "train_pa_one_hot =pa_tokenizer.texts_to_matrix(train_pa_series)\n",
    "test_pa_one_hot =pa_tokenizer.texts_to_matrix(test_pa_series)\n",
    "\n",
    "\n",
    "#INV\n",
    "inv_tokenizer = Tokenizer(num_words=pa_inv_vocab_size,  filters='!\"#$%&()*+,./:;<=>?@[\\]^`{|}~', lower=True, split=' ', char_level=False, oov_token=None)\n",
    "inv_tokenizer.fit_on_texts(train_inv_series)\n",
    "train_inv_one_hot =inv_tokenizer.texts_to_matrix(train_inv_series)\n",
    "test_inv_one_hot =inv_tokenizer.texts_to_matrix(test_inv_series)\n",
    "\n",
    "\n",
    "print('Found %s words in PA' % len(pa_tokenizer.word_index))\n",
    "print('Found %s words in INV' % len(inv_tokenizer.word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6min 28s, sys: 3.59 s, total: 6min 31s\n",
      "Wall time: 6min 32s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "#Title\n",
    "TI_tokenizer = Tokenizer(num_words=50000,  filters='!\"#$%&()*+,./:;<=>?@[\\]^`{|}~_', lower=True, split=' ', char_level=False, oov_token=None)\n",
    "TI_tokenizer.fit_on_texts(train_TI)\n",
    "encoded_train_TI = TI_tokenizer.texts_to_sequences(train_TI)\n",
    "encoded_test_TI = TI_tokenizer.texts_to_sequences(test_TI)\n",
    "#convert all sequences in a list into the same length\n",
    "TI_train = pad_sequences(encoded_train_TI,  maxlen=100, padding='post')\n",
    "TI_test = pad_sequences(encoded_test_TI,  maxlen=100, padding='post')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 s, sys: 101 ms, total: 4.1 s\n",
      "Wall time: 4.11 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# representing the labels/classes in the numeric format by scikit-learn - LabelBinarizer class\n",
    "# Convert 1-dimensional class arrays to n-dimensional(#classes) class matrices\n",
    "encoder = LabelBinarizer()\n",
    "encoder.fit(train_ICM)\n",
    "y_train = encoder.transform(train_ICM)\n",
    "y_test = encoder.transform(test_ICM)\n",
    "\n",
    "#get the unique number of labels in the training set\n",
    "classesList = train_ICM.tolist()\n",
    "classesList =set(classesList)\n",
    "num_classes = len(classesList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def load_embedding_model(filePath):\n",
    "    embeddings_index = dict()\n",
    "    f = open(filePath, encoding='utf8')\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "        \n",
    "    return embeddings_index\n",
    "\n",
    "def create_embedding_matrix(tokenizer, embeddings_index, vocab_size_embbs, dim_size):\n",
    "    embeddings_matrix = np.zeros((vocab_size_embbs, dim_size))\n",
    "    for word, i in tokenizer.word_index.items():\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            embeddings_matrix[i] = embedding_vector[0:dim_size]\n",
    "    \n",
    "    return embeddings_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 29s, sys: 3.33 s, total: 1min 32s\n",
      "Wall time: 1min 32s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## load the whole embedding into memory and get matrix\n",
    "embedding_index = load_embedding_model('../models/w2v/phrase/patWordPhrase2VecModel.txt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.11 s, sys: 758 ms, total: 1.87 s\n",
      "Wall time: 1.87 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "#create TITLE embedding Matrix\n",
    "#vocab_size for embedding\n",
    "vocab_size_embb = len(TI_tokenizer.word_index) + 1\n",
    "\n",
    "TI_embeddings_matrix = create_embedding_matrix(TI_tokenizer,\n",
    "                                              embedding_index,\n",
    "                                              vocab_size_embb,\n",
    "                                              100)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Input, Embedding, BatchNormalization, ELU, Concatenate\n",
    "from keras.layers import LSTM, Conv1D, MaxPooling1D\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.layers.core import Dropout\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.21 s, sys: 10.9 s, total: 16.1 s\n",
      "Wall time: 6.99 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#TITLE \n",
    "sequence_len =100\n",
    "dropout_pct =  0.4\n",
    "\n",
    "TI_embedding_layer_input = Input(shape=(sequence_len,), name='TI_embed_input')\n",
    "TI_embedding_layer = Embedding(input_dim=len(TI_tokenizer.word_index) + 1,\n",
    "                        output_dim=100, # Dimension of the dense embedding\n",
    "                        weights=[TI_embeddings_matrix],\n",
    "                        input_length=100)(TI_embedding_layer_input)\n",
    "\n",
    "lstm_size = 64\n",
    "TI_deep = LSTM(lstm_size,\n",
    "            dropout=dropout_pct,\n",
    "            recurrent_dropout=dropout_pct,\n",
    "            return_sequences=False,\n",
    "            name='LSTM_TI')(TI_embedding_layer)\n",
    "\n",
    "TI_deep = Dense(300, activation=None)(TI_deep)\n",
    "TI_deep = Dropout(dropout_pct)(TI_deep)\n",
    "TI_deep = BatchNormalization()(TI_deep)\n",
    "TI_deep = ELU()(TI_deep)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pa_input and inv_input layers are finished\n"
     ]
    }
   ],
   "source": [
    "dropout_pct =  0.4\n",
    "pa_input = Input(shape=(train_pa_one_hot.shape[1],), name='pa_input') \n",
    "pas = Dense(32,input_dim=train_pa_one_hot.shape[1], activation=None)(pa_input) \n",
    "pas = Dropout(dropout_pct)(pas)\n",
    "pas = BatchNormalization()(pas)\n",
    "pas = ELU()(pas)\n",
    "\n",
    "#inv\n",
    "inv_input = Input(shape=(train_inv_one_hot.shape[1],), name='inv_input') \n",
    "invs = Dense(32,input_dim=train_inv_one_hot.shape[1], activation=None)(pa_input) \n",
    "invs = Dropout(dropout_pct)(invs)\n",
    "invs = BatchNormalization()(invs)\n",
    "\n",
    "print('pa_input and inv_input layers are finished')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "TI_embed_input (InputLayer)  (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 100, 100)          129596000 \n",
      "_________________________________________________________________\n",
      "LSTM_TI (LSTM)               (None, 64)                42240     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 300)               19500     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 300)               1200      \n",
      "_________________________________________________________________\n",
      "elu_1 (ELU)                  (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               38528     \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "elu_3 (ELU)                  (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 42)                5418      \n",
      "=================================================================\n",
      "Total params: 129,703,398\n",
      "Trainable params: 129,702,542\n",
      "Non-trainable params: 856\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import keras_metrics as km\n",
    "#contacting two input models\n",
    "#model_inputs_to_concat = [TI_deep, AB_deep, TECHF_deep, BACKG_deep, SUMM_deep, CLMS_deep] #invs , pas, invs\n",
    "#final_layer =  Concatenate(name='concatenated_layer')(model_inputs_to_concat)\n",
    "\n",
    "output = Dense(128, activation=None)(TI_deep)\n",
    "output = Dropout(dropout_pct)(output)\n",
    "output = BatchNormalization()(output)\n",
    "output = ELU()(output)\n",
    "output = Dense(num_classes, activation='softmax')(output)\n",
    "\n",
    "model = Model(inputs=[TI_embedding_layer_input\n",
    "                     ],\n",
    "              outputs=output, name='model')\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer='adam',\n",
    "                      metrics=['accuracy', km.categorical_precision(), km.categorical_recall()])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/users/mso/anaconda3/envs/msoenv/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:109: UserWarning: Converting sparse IndexedSlices to a dense Tensor with 129596000 elements. This may consume a large amount of memory.\n",
      "  num_elements)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 363353 samples, validate on 40373 samples\n",
      "Epoch 1/20\n",
      "363353/363353 [==============================] - 125s 343us/step - loss: 1.9834 - acc: 0.4814 - precision: 0.2000 - recall: 4.8670e-04 - val_loss: 1.5555 - val_acc: 0.5555 - val_precision: 0.3750 - val_recall: 0.0046\n",
      "Epoch 2/20\n",
      "363353/363353 [==============================] - 115s 315us/step - loss: 1.5855 - acc: 0.5576 - precision: 0.4175 - recall: 0.0140 - val_loss: 1.4328 - val_acc: 0.5763 - val_precision: 0.3781 - val_recall: 0.3040\n",
      "Epoch 3/20\n",
      "363353/363353 [==============================] - 115s 315us/step - loss: 1.4811 - acc: 0.5739 - precision: 0.4957 - recall: 0.0655 - val_loss: 1.3971 - val_acc: 0.5735 - val_precision: 0.4179 - val_recall: 0.2948\n",
      "Epoch 4/20\n",
      "363353/363353 [==============================] - 115s 317us/step - loss: 1.4289 - acc: 0.5833 - precision: 0.4855 - recall: 0.0923 - val_loss: 1.3671 - val_acc: 0.5818 - val_precision: 0.4130 - val_recall: 0.3333\n",
      "Epoch 5/20\n",
      "363353/363353 [==============================] - 116s 319us/step - loss: 1.3925 - acc: 0.5909 - precision: 0.5066 - recall: 0.1252 - val_loss: 1.3395 - val_acc: 0.5877 - val_precision: 0.4444 - val_recall: 0.3765\n",
      "Epoch 6/20\n",
      "363353/363353 [==============================] - 118s 323us/step - loss: 1.3678 - acc: 0.5963 - precision: 0.5006 - recall: 0.1329 - val_loss: 1.3268 - val_acc: 0.5887 - val_precision: 0.4073 - val_recall: 0.4506\n",
      "Epoch 7/20\n",
      "363353/363353 [==============================] - 115s 317us/step - loss: 1.3487 - acc: 0.6009 - precision: 0.5140 - recall: 0.1580 - val_loss: 1.3013 - val_acc: 0.5958 - val_precision: 0.4104 - val_recall: 0.4630\n",
      "Epoch 8/20\n",
      "363353/363353 [==============================] - 114s 314us/step - loss: 1.3308 - acc: 0.6053 - precision: 0.5260 - recall: 0.1591 - val_loss: 1.3043 - val_acc: 0.5936 - val_precision: 0.4264 - val_recall: 0.4383\n",
      "Epoch 9/20\n",
      "363353/363353 [==============================] - 114s 314us/step - loss: 1.3170 - acc: 0.6082 - precision: 0.5281 - recall: 0.1752 - val_loss: 1.2888 - val_acc: 0.5971 - val_precision: 0.4380 - val_recall: 0.4198\n",
      "Epoch 10/20\n",
      "363353/363353 [==============================] - 113s 311us/step - loss: 1.3043 - acc: 0.6109 - precision: 0.5251 - recall: 0.1750 - val_loss: 1.2724 - val_acc: 0.6016 - val_precision: 0.4221 - val_recall: 0.4429\n",
      "Epoch 11/20\n",
      "363353/363353 [==============================] - 115s 317us/step - loss: 1.2910 - acc: 0.6138 - precision: 0.5392 - recall: 0.1918 - val_loss: 1.2576 - val_acc: 0.6096 - val_precision: 0.4269 - val_recall: 0.4414\n",
      "Epoch 12/20\n",
      "363353/363353 [==============================] - 114s 313us/step - loss: 1.2826 - acc: 0.6161 - precision: 0.5329 - recall: 0.1986 - val_loss: 1.2648 - val_acc: 0.6053 - val_precision: 0.4430 - val_recall: 0.4074\n",
      "Epoch 13/20\n",
      "363353/363353 [==============================] - 118s 324us/step - loss: 1.2722 - acc: 0.6191 - precision: 0.5533 - recall: 0.2054 - val_loss: 1.2553 - val_acc: 0.6096 - val_precision: 0.4620 - val_recall: 0.4506\n",
      "Epoch 14/20\n",
      "363353/363353 [==============================] - 115s 317us/step - loss: 1.2641 - acc: 0.6206 - precision: 0.5571 - recall: 0.2201 - val_loss: 1.2558 - val_acc: 0.6087 - val_precision: 0.4467 - val_recall: 0.4336\n",
      "Epoch 15/20\n",
      "363353/363353 [==============================] - 116s 320us/step - loss: 1.2563 - acc: 0.6231 - precision: 0.5387 - recall: 0.2080 - val_loss: 1.2442 - val_acc: 0.6115 - val_precision: 0.4487 - val_recall: 0.4660\n",
      "Epoch 16/20\n",
      "363353/363353 [==============================] - 114s 315us/step - loss: 1.2498 - acc: 0.6235 - precision: 0.5633 - recall: 0.2223 - val_loss: 1.2278 - val_acc: 0.6148 - val_precision: 0.4317 - val_recall: 0.5123\n",
      "Epoch 17/20\n",
      "363353/363353 [==============================] - 114s 315us/step - loss: 1.2398 - acc: 0.6266 - precision: 0.5688 - recall: 0.2380 - val_loss: 1.2258 - val_acc: 0.6149 - val_precision: 0.4758 - val_recall: 0.4552\n",
      "Epoch 18/20\n",
      "363353/363353 [==============================] - 113s 312us/step - loss: 1.2357 - acc: 0.6266 - precision: 0.5556 - recall: 0.2279 - val_loss: 1.2118 - val_acc: 0.6193 - val_precision: 0.4376 - val_recall: 0.4815\n",
      "Epoch 19/20\n",
      "363353/363353 [==============================] - 116s 320us/step - loss: 1.2280 - acc: 0.6295 - precision: 0.5705 - recall: 0.2390 - val_loss: 1.2315 - val_acc: 0.6129 - val_precision: 0.4664 - val_recall: 0.4290\n",
      "Epoch 20/20\n",
      "363353/363353 [==============================] - 114s 313us/step - loss: 1.2237 - acc: 0.6303 - precision: 0.5725 - recall: 0.2435 - val_loss: 1.2158 - val_acc: 0.6167 - val_precision: 0.5000 - val_recall: 0.4306\n",
      "CPU times: user 1h 1s, sys: 8min 29s, total: 1h 8min 31s\n",
      "Wall time: 38min 41s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "batch_size= 500 \n",
    "num_epochs = 20\n",
    "\n",
    "history = model.fit(x={'TI_embed_input': TI_train\n",
    "             \n",
    "            },\n",
    "          y=y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=num_epochs,\n",
    "          validation_data=\n",
    "          ({'TI_embed_input': TI_test\n",
    "            \n",
    "            },\n",
    "           y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_circles\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from matplotlib import pyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras_metrics as km"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "TI_embed_input (InputLayer)     (None, 100)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 100, 100)     129596000   TI_embed_input[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "LSTM_TI (LSTM)                  (None, 64)           42240       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "pa_input (InputLayer)           (None, 2000)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 300)          19500       LSTM_TI[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 32)           64032       pa_input[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 300)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 32)           0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 32)           64032       pa_input[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 300)          1200        dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 32)           128         dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 32)           0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "elu_1 (ELU)                     (None, 300)          0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "elu_2 (ELU)                     (None, 32)           0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 32)           128         dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenated_layer (Concatenate (None, 364)          0           elu_1[0][0]                      \n",
      "                                                                 elu_2[0][0]                      \n",
      "                                                                 batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 128)          46720       concatenated_layer[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 128)          0           dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 128)          512         dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "elu_4 (ELU)                     (None, 128)          0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 42)           5418        elu_4[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 129,839,910\n",
      "Trainable params: 129,838,926\n",
      "Non-trainable params: 984\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import keras_metrics as km\n",
    "\n",
    "#contacting two input models\n",
    "model_inputs_to_concat = [TI_deep, pas, invs] #invs , pas, invs\n",
    "final_layer =  Concatenate(name='concatenated_layer')(model_inputs_to_concat)\n",
    "\n",
    "output = Dense(128, activation=None)(final_layer)\n",
    "output = Dropout(dropout_pct)(output)\n",
    "output = BatchNormalization()(output)\n",
    "output = ELU()(output)\n",
    "output = Dense(num_classes, activation='softmax')(output)\n",
    "\n",
    "model2 =Model(inputs=[ TI_embedding_layer_input,\n",
    "                     pa_input,\n",
    "                      inv_input],\n",
    "              outputs=output, name='model')\n",
    "model2.compile(loss='categorical_crossentropy',\n",
    "                      optimizer='adam',\n",
    "                       metrics=['accuracy', km.categorical_precision(), km.categorical_recall()])\n",
    "model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/users/mso/anaconda3/envs/msoenv/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:109: UserWarning: Converting sparse IndexedSlices to a dense Tensor with 129596000 elements. This may consume a large amount of memory.\n",
      "  num_elements)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 363353 samples, validate on 40373 samples\n",
      "Epoch 1/20\n",
      "363353/363353 [==============================] - 130s 357us/step - loss: 1.3956 - acc: 0.6109 - precision: 0.5416 - recall: 0.2229 - val_loss: 1.2254 - val_acc: 0.6267 - val_precision: 0.4509 - val_recall: 0.4676\n",
      "Epoch 2/20\n",
      "363353/363353 [==============================] - 123s 339us/step - loss: 1.2261 - acc: 0.6323 - precision: 0.5826 - recall: 0.2665 - val_loss: 1.2141 - val_acc: 0.6234 - val_precision: 0.4640 - val_recall: 0.4676\n",
      "Epoch 3/20\n",
      "363353/363353 [==============================] - 126s 347us/step - loss: 1.1995 - acc: 0.6379 - precision: 0.5806 - recall: 0.2787 - val_loss: 1.2169 - val_acc: 0.6227 - val_precision: 0.4678 - val_recall: 0.5046\n",
      "Epoch 4/20\n",
      "363353/363353 [==============================] - 125s 343us/step - loss: 1.1852 - acc: 0.6408 - precision: 0.5991 - recall: 0.2844 - val_loss: 1.2207 - val_acc: 0.6204 - val_precision: 0.4587 - val_recall: 0.5231\n",
      "Epoch 5/20\n",
      "363353/363353 [==============================] - 121s 333us/step - loss: 1.1740 - acc: 0.6428 - precision: 0.6013 - recall: 0.2975 - val_loss: 1.2086 - val_acc: 0.6212 - val_precision: 0.4706 - val_recall: 0.5185\n",
      "Epoch 6/20\n",
      "363353/363353 [==============================] - 122s 335us/step - loss: 1.1664 - acc: 0.6455 - precision: 0.6130 - recall: 0.3097 - val_loss: 1.2022 - val_acc: 0.6261 - val_precision: 0.4809 - val_recall: 0.4861\n",
      "Epoch 7/20\n",
      "363353/363353 [==============================] - 123s 338us/step - loss: 1.1609 - acc: 0.6457 - precision: 0.6141 - recall: 0.3073 - val_loss: 1.2120 - val_acc: 0.6235 - val_precision: 0.5017 - val_recall: 0.4522\n",
      "Epoch 8/20\n",
      "363353/363353 [==============================] - 122s 336us/step - loss: 1.1531 - acc: 0.6478 - precision: 0.6068 - recall: 0.3144 - val_loss: 1.2049 - val_acc: 0.6211 - val_precision: 0.4760 - val_recall: 0.5046\n",
      "Epoch 9/20\n",
      "363353/363353 [==============================] - 122s 335us/step - loss: 1.1466 - acc: 0.6492 - precision: 0.6064 - recall: 0.3087 - val_loss: 1.2008 - val_acc: 0.6256 - val_precision: 0.4424 - val_recall: 0.5448\n",
      "Epoch 10/20\n",
      "363353/363353 [==============================] - 121s 334us/step - loss: 1.1421 - acc: 0.6497 - precision: 0.6097 - recall: 0.3160 - val_loss: 1.1901 - val_acc: 0.6310 - val_precision: 0.4727 - val_recall: 0.4815\n",
      "Epoch 11/20\n",
      "363353/363353 [==============================] - 123s 340us/step - loss: 1.1358 - acc: 0.6515 - precision: 0.6221 - recall: 0.3295 - val_loss: 1.2046 - val_acc: 0.6240 - val_precision: 0.4926 - val_recall: 0.4630\n",
      "Epoch 12/20\n",
      "363353/363353 [==============================] - 123s 339us/step - loss: 1.1321 - acc: 0.6523 - precision: 0.6078 - recall: 0.3224 - val_loss: 1.1939 - val_acc: 0.6288 - val_precision: 0.4929 - val_recall: 0.4830\n",
      "Epoch 13/20\n",
      "363353/363353 [==============================] - 122s 335us/step - loss: 1.1250 - acc: 0.6536 - precision: 0.6152 - recall: 0.3250 - val_loss: 1.2038 - val_acc: 0.6234 - val_precision: 0.5041 - val_recall: 0.4707\n",
      "Epoch 14/20\n",
      "363353/363353 [==============================] - 122s 336us/step - loss: 1.1207 - acc: 0.6554 - precision: 0.6185 - recall: 0.3323 - val_loss: 1.1981 - val_acc: 0.6272 - val_precision: 0.5196 - val_recall: 0.4290\n",
      "Epoch 15/20\n",
      "363353/363353 [==============================] - 121s 334us/step - loss: 1.1185 - acc: 0.6559 - precision: 0.6255 - recall: 0.3404 - val_loss: 1.1923 - val_acc: 0.6274 - val_precision: 0.4852 - val_recall: 0.4552\n",
      "Epoch 16/20\n",
      "363353/363353 [==============================] - 121s 332us/step - loss: 1.1134 - acc: 0.6570 - precision: 0.6313 - recall: 0.3431 - val_loss: 1.1964 - val_acc: 0.6277 - val_precision: 0.4862 - val_recall: 0.5154\n",
      "Epoch 17/20\n",
      "363353/363353 [==============================] - 122s 335us/step - loss: 1.1094 - acc: 0.6585 - precision: 0.6353 - recall: 0.3516 - val_loss: 1.2008 - val_acc: 0.6265 - val_precision: 0.4865 - val_recall: 0.5015\n",
      "Epoch 18/20\n",
      "363353/363353 [==============================] - 122s 336us/step - loss: 1.1042 - acc: 0.6597 - precision: 0.6310 - recall: 0.3410 - val_loss: 1.2036 - val_acc: 0.6263 - val_precision: 0.4640 - val_recall: 0.5463\n",
      "Epoch 19/20\n",
      "363353/363353 [==============================] - 122s 336us/step - loss: 1.1010 - acc: 0.6608 - precision: 0.6374 - recall: 0.3491 - val_loss: 1.2098 - val_acc: 0.6235 - val_precision: 0.4807 - val_recall: 0.5201\n",
      "Epoch 20/20\n",
      "363353/363353 [==============================] - 121s 333us/step - loss: 1.0985 - acc: 0.6609 - precision: 0.6363 - recall: 0.3472 - val_loss: 1.2063 - val_acc: 0.6269 - val_precision: 0.4828 - val_recall: 0.5201\n",
      "CPU times: user 1h 2min 45s, sys: 8min 33s, total: 1h 11min 19s\n",
      "Wall time: 41min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "batch_size= 500 \n",
    "num_epochs = 20\n",
    "\n",
    "\n",
    "history2 = model2.fit(x={'TI_embed_input': TI_train,\n",
    "             'pa_input': train_pa_one_hot,\n",
    "             'inv_input': train_inv_one_hot\n",
    "            },\n",
    "          y=y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=num_epochs,\n",
    "          validation_data=\n",
    "          ({'TI_embed_input': TI_test,\n",
    "            'pa_input': test_pa_one_hot,\n",
    "            'inv_input': test_inv_one_hot\n",
    "            },\n",
    "           y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}