VAE-LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.layers import Bidirectional, Dense, Embedding, Concatenate, Dot, Flatten, Input, Lambda, LSTM, merge, GlobalAveragePooling1D, RepeatVector, TimeDistributed, Layer, Activation, Dropout\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers.advanced_activations import ELU\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.optimizers import Adam, SGD, RMSprop\n",
    "from keras import objectives\n",
    "from keras import backend as K\n",
    "from keras.models import Model\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import codecs\n",
    "import csv\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 256\n",
    "max_len = 10\n",
    "emb_dim = 50\n",
    "NB_WORDS = 50000\n",
    "latent_dim = 50\n",
    "intermediate_dim = 96\n",
    "epsilon_std = 1.0\n",
    "num_sampled=500\n",
    "act = ELU()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18.9 s, sys: 2.68 s, total: 21.6 s\n",
      "Wall time: 41.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import pandas as pd\n",
    "\n",
    "TRAIN_DATA_FILE = '/data/chzho/deepqts/train_data/unifiedclick/join_oneyearsample_2B_training_all_top10'\n",
    "num_read_row = 1000000\n",
    "df = pd.read_csv(TRAIN_DATA_FILE, sep=\"\\t\", usecols=[0,1,3], names=['label', 'q', 'd'], header=None , error_bad_lines=False, nrows=num_read_row)\n",
    "df = df.dropna()\n",
    "\n",
    "TEST_DATA_FILE = '/data/chzho/deepqts/test_data/uhrs/unified/uhrs_do_10'\n",
    "df_qd = pd.read_csv(TEST_DATA_FILE, sep=\"\\t\", usecols=[0,1,3,5], names=['label', 'q', 'd', 'market'], header=None , error_bad_lines=False)\n",
    "df_qd = df_qd.dropna()\n",
    "df_qd = df_qd[df_qd.market == \"en-US\"]\n",
    "\n",
    "TEST_DATA_FILE = '/data/chzho/deepqts/test_data/julyflower/julyflower_original.tsv'\n",
    "df_qq = pd.read_csv(TEST_DATA_FILE, sep=\"\\t\", names=['q', 'd', 'label'], header=None , error_bad_lines=False)\n",
    "df_qq = df_qq.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "TRAIN_DATA_FILE = '/data/t-mipha/data/agi_encoder/v4/universal/CLICKED_QQ_EN_universal_train_1M.txt'\n",
    "df_qq_click = pd.read_csv(TRAIN_DATA_FILE, usecols=[0,1], names=[\"q\", \"sim_q\"], sep=\"\\t\", header=None, error_bad_lines=False)\n",
    "df_qq_click = df_qq_click.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "q1_qq_click_raw = df_qq_click.q.tolist()\n",
    "q2_qq_click_raw = [i.split(\"<sep>\")[0] for i in df_qq_click.sim_q.tolist()]\n",
    "y_qq_click_train = np.ones(len(df_qq_click))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "texts = q1_qq_click_raw + q2_qq_click_raw + df_qd.q.tolist() + df_qd.d.tolist() + df_qq.q.tolist() + df_qq.d.tolist() + df.q.tolist() + df.d.tolist()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 210261 unique tokens\n",
      "Number of Vocab: 200001\n",
      "CPU times: user 14.5 s, sys: 172 ms, total: 14.7 s\n",
      "Wall time: 14.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tokenizer = Tokenizer(NB_WORDS)\n",
    "tokenizer.fit_on_texts(texts)\n",
    "word_index = tokenizer.word_index #the dict values start from 1 so this is fine with zeropadding\n",
    "index2word = {v: k for k, v in word_index.items()}\n",
    "print('Found %s unique tokens' % len(word_index))\n",
    "NB_WORDS = (min(tokenizer.num_words, len(word_index)) + 1 ) #+1 for zero padding\n",
    "print('Number of Vocab: %d' % NB_WORDS)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "uns_q = pad_sequences(tokenizer.texts_to_sequences(df.q.tolist()), maxlen=max_len)\n",
    "uns_d = pad_sequences(tokenizer.texts_to_sequences(df.d.tolist()), maxlen=max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "def create_lstm_vae(input_dim, \n",
    "    timesteps, \n",
    "    batch_size, \n",
    "    intermediate_dim, \n",
    "    latent_dim,\n",
    "    epsilon_std=1.):\n",
    "\n",
    "    \"\"\"\n",
    "    Creates an LSTM Variational Autoencoder (VAE). Returns VAE, Encoder, Generator. \n",
    "    # Arguments\n",
    "        input_dim: int.\n",
    "        timesteps: int, input timestep dimension.\n",
    "        batch_size: int.\n",
    "        intermediate_dim: int, output shape of LSTM. \n",
    "        latent_dim: int, latent z-layer shape. \n",
    "        epsilon_std: float, z-layer sigma.\n",
    "    # References\n",
    "        - [Building Autoencoders in Keras](https://blog.keras.io/building-autoencoders-in-keras.html)\n",
    "        - [Generating sentences from a continuous space](https://arxiv.org/abs/1511.06349)\n",
    "    \"\"\"\n",
    "    x = Input(shape=(timesteps, input_dim,))\n",
    "\n",
    "    # LSTM encoding\n",
    "    h = LSTM(intermediate_dim)(x)\n",
    "\n",
    "    # VAE Z layer\n",
    "    z_mean = Dense(latent_dim)(h)\n",
    "    z_log_sigma = Dense(latent_dim)(h)\n",
    "    \n",
    "    def sampling(args):\n",
    "        z_mean, z_log_sigma = args\n",
    "        epsilon = K.random_normal(shape=(batch_size, latent_dim),\n",
    "                                  mean=0., stddev=epsilon_std)\n",
    "        return z_mean + z_log_sigma * epsilon\n",
    "\n",
    "    # note that \"output_shape\" isn't necessary with the TensorFlow backend\n",
    "    # so you could write `Lambda(sampling)([z_mean, z_log_sigma])`\n",
    "    z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_sigma])\n",
    "    \n",
    "    # decoded LSTM layer\n",
    "    decoder_h = LSTM(intermediate_dim, return_sequences=True)\n",
    "    decoder_mean = LSTM(input_dim, return_sequences=True)\n",
    "\n",
    "    h_decoded = RepeatVector(timesteps)(z)\n",
    "    h_decoded = decoder_h(h_decoded)\n",
    "\n",
    "    # decoded layer\n",
    "    x_decoded_mean = decoder_mean(h_decoded)\n",
    "    \n",
    "    # end-to-end autoencoder\n",
    "    vae = Model(x, x_decoded_mean)\n",
    "\n",
    "    # encoder, from inputs to latent space\n",
    "    encoder = Model(x, [z_mean, z_log_sigma, z])\n",
    "\n",
    "    # generator, from latent space to reconstructed inputs\n",
    "    decoder_input = Input(shape=(latent_dim,))\n",
    "\n",
    "    _h_decoded = RepeatVector(timesteps)(decoder_input)\n",
    "    _h_decoded = decoder_h(_h_decoded)\n",
    "\n",
    "    _x_decoded_mean = decoder_mean(_h_decoded)\n",
    "    generator = Model(decoder_input, _x_decoded_mean)\n",
    "    \n",
    "    def vae_loss(x, x_decoded_mean):\n",
    "        xent_loss = objectives.mse(x, x_decoded_mean)\n",
    "        kl_loss = - 0.5 * K.mean(1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma))\n",
    "        loss = xent_loss + kl_loss\n",
    "        return loss\n",
    "\n",
    "    vae.compile(optimizer='rmsprop', loss=vae_loss)\n",
    "    \n",
    "    return vae, encoder, generator\n",
    "vae, encoder, decoder = create_lstm_vae(emb_dim, max_len, batch_size, intermediate_dim, latent_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_w2v():\n",
    "    GLOVE_EMBEDDING = '/home/t-jamano/data/glove/glove.6B.50d.txt'\n",
    "    embeddings_index = {}\n",
    "    f = open(GLOVE_EMBEDDING, encoding='utf8')\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "\n",
    "    print('Found %s word vectors.' % len(embeddings_index))\n",
    "\n",
    "\n",
    "    glove_embedding_matrix = np.zeros((NB_WORDS, emb_dim))\n",
    "    for word, i in word_index.items():\n",
    "        if i < NB_WORDS:\n",
    "            embedding_vector = embeddings_index.get(word)\n",
    "            if embedding_vector is not None:\n",
    "                # words not found in embedding index will be the word embedding of 'unk'.\n",
    "                glove_embedding_matrix[i] = embedding_vector\n",
    "            else:\n",
    "                glove_embedding_matrix[i] = embeddings_index.get('unk')\n",
    "    print('Null word embeddings: %d' % np.sum(np.sum(glove_embedding_matrix, axis=1) == 0))\n",
    "    \n",
    "    w2v = Sequential()\n",
    "    w2v.add(Embedding(NB_WORDS, emb_dim, input_length=max_len, weights=[glove_embedding_matrix]))\n",
    "    \n",
    "    return w2v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 400000 word vectors.\n",
      "Null word embeddings: 1\n"
     ]
    }
   ],
   "source": [
    "w2v = get_w2v()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 5)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uns_q.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "uns_q = w2v.predict(uns_q)\n",
    "uns_d = w2v.predict(uns_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      " - 19s - loss: 0.2456\n",
      "Epoch 2/20\n",
      " - 17s - loss: 0.2148\n",
      "Epoch 3/20\n",
      " - 17s - loss: 0.2012\n",
      "Epoch 4/20\n",
      " - 17s - loss: 0.1913\n",
      "Epoch 5/20\n",
      " - 17s - loss: 0.1841\n",
      "Epoch 6/20\n",
      " - 17s - loss: 0.1786\n",
      "Epoch 7/20\n",
      " - 17s - loss: 0.1739\n",
      "Epoch 8/20\n",
      " - 17s - loss: 0.1702\n",
      "Epoch 9/20\n",
      " - 17s - loss: 0.1672\n",
      "Epoch 10/20\n",
      " - 17s - loss: 0.1644\n",
      "Epoch 11/20\n",
      " - 17s - loss: 0.1620\n",
      "Epoch 12/20\n",
      " - 17s - loss: 0.1597\n",
      "Epoch 13/20\n",
      " - 17s - loss: 0.1577\n",
      "Epoch 14/20\n",
      " - 17s - loss: 0.1559\n",
      "Epoch 15/20\n",
      " - 17s - loss: 0.1542\n",
      "Epoch 16/20\n",
      " - 17s - loss: 0.1527\n",
      "Epoch 17/20\n",
      " - 17s - loss: 0.1513\n",
      "Epoch 18/20\n",
      " - 17s - loss: 0.1500\n",
      "Epoch 19/20\n",
      " - 17s - loss: 0.1489\n",
      "Epoch 20/20\n",
      " - 17s - loss: 0.1477\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fe3c6fab518>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vae.fit(uns_q, uns_q,  verbose=2, batch_size=batch_size, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      " - 17s - loss: 0.2372\n",
      "Epoch 2/20\n",
      " - 17s - loss: 0.2325\n",
      "Epoch 3/20\n",
      " - 17s - loss: 0.2289\n",
      "Epoch 4/20\n",
      " - 17s - loss: 0.2258\n",
      "Epoch 5/20\n",
      " - 17s - loss: 0.2229\n",
      "Epoch 6/20\n",
      " - 17s - loss: 0.2201\n",
      "Epoch 7/20\n",
      " - 17s - loss: 0.2175\n",
      "Epoch 8/20\n",
      " - 17s - loss: 0.2151\n",
      "Epoch 9/20\n",
      " - 17s - loss: 0.2130\n",
      "Epoch 10/20\n",
      " - 17s - loss: 0.2109\n",
      "Epoch 11/20\n",
      " - 17s - loss: 0.2089\n",
      "Epoch 12/20\n",
      " - 17s - loss: 0.2070\n",
      "Epoch 13/20\n",
      " - 17s - loss: 0.2054\n",
      "Epoch 14/20\n",
      " - 17s - loss: 0.2039\n",
      "Epoch 15/20\n",
      " - 17s - loss: 0.2025\n",
      "Epoch 16/20\n",
      " - 17s - loss: 0.2012\n",
      "Epoch 17/20\n",
      " - 17s - loss: 0.1999\n",
      "Epoch 18/20\n",
      " - 17s - loss: 0.1987\n",
      "Epoch 19/20\n",
      " - 17s - loss: 0.1974\n",
      "Epoch 20/20\n",
      " - 17s - loss: 0.1961\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fe3c6f98470>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vae.fit(uns_d, uns_d,  verbose=2, batch_size=batch_size, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "q_qq = pad_sequences(tokenizer.texts_to_sequences(df_qq.q.tolist()), maxlen=max_len)\n",
    "d_qq = pad_sequences(tokenizer.texts_to_sequences(df_qq.d.tolist()), maxlen=max_len)\n",
    "# q_qd = pad_sequences(tokenizer.texts_to_sequences(df_qd.q.tolist()), maxlen=max_len)\n",
    "# d_qd = pad_sequences(tokenizer.texts_to_sequences(df_qd.d.tolist()), maxlen=max_len)\n",
    "\n",
    "\n",
    "# first encode with W2V\n",
    "q_qq = w2v.predict(q_qq)\n",
    "d_qq = w2v.predict(d_qq)\n",
    "# q_qd = w2v.predict(q_qd)\n",
    "# d_qd = w2v.predict(d_qd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "class W2V_MLP():\n",
    "    def __init__(self, max_len, input_dim):\n",
    "\n",
    "        que_input = Input(shape=(max_len, input_dim,))\n",
    "        doc_input = Input(shape=(max_len, input_dim,))\n",
    "        \n",
    "        x = GlobalAveragePooling1D()(que_input)\n",
    "        y = GlobalAveragePooling1D()(doc_input)\n",
    "\n",
    "        concat = merge([x, y], mode=\"concat\")\n",
    "\n",
    "\n",
    "        pred = Dense(1, activation='sigmoid')(concat)\n",
    "\n",
    "        self.model = Model(input=[que_input, doc_input], output=pred)\n",
    "        self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "class AVGPolling():\n",
    "    def __init__(self, max_len, emb_dim):\n",
    "        x_input = Input(shape=(max_len, emb_dim,))\n",
    "        avg = GlobalAveragePooling1D()(x_input)\n",
    "        self.model = Model(input=x_input, output=avg)\n",
    "\n",
    "\n",
    "class LSTM():\n",
    "    def __init__(self, max_len, input_dim):\n",
    "        \n",
    "        que_input = Input(shape=(max_len, input_dim,))\n",
    "        doc_input = Input(shape=(max_len, input_dim,))\n",
    "        \n",
    "        emb = Embedding()\n",
    "        \n",
    "        x = GlobalAveragePooling1D()(que_input)\n",
    "        y = GlobalAveragePooling1D()(doc_input)\n",
    "\n",
    "        concat = merge([x, y], mode=\"concat\")\n",
    "\n",
    "\n",
    "        pred = Dense(1, activation='sigmoid')(concat)\n",
    "\n",
    "        self.model = Model(input=[que_input, doc_input], output=pred)\n",
    "        self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# build a model to project sentences on the latent space# build \n",
    "# encoder = Model(x, z)\n",
    "enc_q_qq = encoder.predict(q_qq)\n",
    "enc_d_qq = encoder.predict(d_qq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:9: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "/home/t-jamano/.local/lib/python3.6/site-packages/keras/legacy/layers.py:458: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  name=name)\n",
      "/usr/local/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:14: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=Tensor(\"de...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 213 samples, validate on 213 samples\n",
      "Epoch 1/20\n",
      " - 1s - loss: 0.6945 - acc: 0.4977 - val_loss: 0.6856 - val_acc: 0.6197\n",
      "Epoch 2/20\n",
      " - 0s - loss: 0.6914 - acc: 0.5258 - val_loss: 0.6841 - val_acc: 0.6150\n",
      "Epoch 3/20\n",
      " - 0s - loss: 0.6885 - acc: 0.5587 - val_loss: 0.6828 - val_acc: 0.6291\n",
      "Epoch 4/20\n",
      " - 0s - loss: 0.6860 - acc: 0.5446 - val_loss: 0.6812 - val_acc: 0.6385\n",
      "Epoch 5/20\n",
      " - 0s - loss: 0.6832 - acc: 0.5587 - val_loss: 0.6800 - val_acc: 0.6479\n",
      "Epoch 6/20\n",
      " - 0s - loss: 0.6808 - acc: 0.5587 - val_loss: 0.6789 - val_acc: 0.6197\n",
      "Epoch 7/20\n",
      " - 0s - loss: 0.6784 - acc: 0.5822 - val_loss: 0.6776 - val_acc: 0.6056\n",
      "Epoch 8/20\n",
      " - 0s - loss: 0.6760 - acc: 0.6103 - val_loss: 0.6766 - val_acc: 0.6244\n",
      "Epoch 9/20\n",
      " - 0s - loss: 0.6738 - acc: 0.6103 - val_loss: 0.6757 - val_acc: 0.6197\n",
      "Epoch 10/20\n",
      " - 0s - loss: 0.6716 - acc: 0.6244 - val_loss: 0.6747 - val_acc: 0.6150\n",
      "Epoch 11/20\n",
      " - 0s - loss: 0.6696 - acc: 0.6150 - val_loss: 0.6738 - val_acc: 0.6103\n",
      "Epoch 12/20\n",
      " - 0s - loss: 0.6675 - acc: 0.6291 - val_loss: 0.6731 - val_acc: 0.6103\n",
      "Epoch 13/20\n",
      " - 0s - loss: 0.6656 - acc: 0.6338 - val_loss: 0.6722 - val_acc: 0.6009\n",
      "Epoch 14/20\n",
      " - 0s - loss: 0.6636 - acc: 0.6291 - val_loss: 0.6714 - val_acc: 0.6056\n",
      "Epoch 15/20\n",
      " - 0s - loss: 0.6618 - acc: 0.6432 - val_loss: 0.6707 - val_acc: 0.6009\n",
      "Epoch 16/20\n",
      " - 0s - loss: 0.6600 - acc: 0.6432 - val_loss: 0.6700 - val_acc: 0.6056\n",
      "Epoch 17/20\n",
      " - 0s - loss: 0.6581 - acc: 0.6479 - val_loss: 0.6692 - val_acc: 0.5962\n",
      "Epoch 18/20\n",
      " - 0s - loss: 0.6563 - acc: 0.6573 - val_loss: 0.6685 - val_acc: 0.5915\n",
      "Epoch 19/20\n",
      " - 0s - loss: 0.6547 - acc: 0.6573 - val_loss: 0.6679 - val_acc: 0.5775\n",
      "Epoch 20/20\n",
      " - 0s - loss: 0.6530 - acc: 0.6620 - val_loss: 0.6673 - val_acc: 0.5775\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fe3b2b0b240>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp = MLP(latent_dim)\n",
    "mlp.model.fit([enc_q_qq, enc_d_qq], df_qq.label.values, verbose=2, batch_size=8, validation_split=0.5, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(426, 5, 50)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_qq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def split_data(q, d, df, size, axis=1):\n",
    "    concat = np.concatenate((q, d), axis=axis)\n",
    "    label = df.label.values\n",
    "    return train_test_split(concat, label, test_size=size, random_state=42)\n",
    "    \n",
    "\n",
    "x_train, x_test, y_train, y_test = split_data(enc_q_qq, enc_d_qq, df_qq, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred = mlp.model.predict([x_test[:, :latent_dim], x_test[:, latent_dim:]]).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.72005323868677906"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(y_test, pred, pos_label=1)\n",
    "metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:26: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "/home/t-jamano/.local/lib/python3.6/site-packages/keras/legacy/layers.py:458: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  name=name)\n",
      "/usr/local/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:31: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=Tensor(\"de...)`\n"
     ]
    }
   ],
   "source": [
    "w2v_mlp = W2V_MLP(max_len, emb_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 213 samples, validate on 213 samples\n",
      "Epoch 1/20\n",
      " - 1s - loss: 0.7041 - acc: 0.5634 - val_loss: 0.7223 - val_acc: 0.5164\n",
      "Epoch 2/20\n",
      " - 0s - loss: 0.6918 - acc: 0.5728 - val_loss: 0.7147 - val_acc: 0.5211\n",
      "Epoch 3/20\n",
      " - 0s - loss: 0.6827 - acc: 0.5822 - val_loss: 0.7099 - val_acc: 0.5211\n",
      "Epoch 4/20\n",
      " - 0s - loss: 0.6744 - acc: 0.5728 - val_loss: 0.7058 - val_acc: 0.5117\n",
      "Epoch 5/20\n",
      " - 0s - loss: 0.6672 - acc: 0.5775 - val_loss: 0.7020 - val_acc: 0.5305\n",
      "Epoch 6/20\n",
      " - 0s - loss: 0.6602 - acc: 0.5869 - val_loss: 0.7001 - val_acc: 0.5399\n",
      "Epoch 7/20\n",
      " - 0s - loss: 0.6542 - acc: 0.5869 - val_loss: 0.6975 - val_acc: 0.5446\n",
      "Epoch 8/20\n",
      " - 0s - loss: 0.6482 - acc: 0.6150 - val_loss: 0.6949 - val_acc: 0.5493\n",
      "Epoch 9/20\n",
      " - 0s - loss: 0.6432 - acc: 0.6291 - val_loss: 0.6932 - val_acc: 0.5446\n",
      "Epoch 10/20\n",
      " - 0s - loss: 0.6377 - acc: 0.6620 - val_loss: 0.6918 - val_acc: 0.5493\n",
      "Epoch 11/20\n",
      " - 0s - loss: 0.6337 - acc: 0.6761 - val_loss: 0.6910 - val_acc: 0.5540\n",
      "Epoch 12/20\n",
      " - 0s - loss: 0.6291 - acc: 0.6761 - val_loss: 0.6900 - val_acc: 0.5587\n",
      "Epoch 13/20\n",
      " - 0s - loss: 0.6250 - acc: 0.6761 - val_loss: 0.6891 - val_acc: 0.5728\n",
      "Epoch 14/20\n",
      " - 0s - loss: 0.6214 - acc: 0.6808 - val_loss: 0.6883 - val_acc: 0.5822\n",
      "Epoch 15/20\n",
      " - 0s - loss: 0.6181 - acc: 0.6854 - val_loss: 0.6880 - val_acc: 0.5681\n",
      "Epoch 16/20\n",
      " - 0s - loss: 0.6143 - acc: 0.6808 - val_loss: 0.6870 - val_acc: 0.5822\n",
      "Epoch 17/20\n",
      " - 0s - loss: 0.6112 - acc: 0.6808 - val_loss: 0.6875 - val_acc: 0.5728\n",
      "Epoch 18/20\n",
      " - 0s - loss: 0.6079 - acc: 0.6761 - val_loss: 0.6865 - val_acc: 0.5869\n",
      "Epoch 19/20\n",
      " - 0s - loss: 0.6053 - acc: 0.6854 - val_loss: 0.6866 - val_acc: 0.5869\n",
      "Epoch 20/20\n",
      " - 0s - loss: 0.6024 - acc: 0.6854 - val_loss: 0.6861 - val_acc: 0.5915\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fe3b106b4e0>"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_mlp.model.fit([q_qq, d_qq], df_qq.label.values, verbose=2, batch_size=8, validation_split=0.5, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = split_data(q_qq, d_qq, df_qq, 0.5, 2)\n",
    "\n",
    "pred = w2v_mlp.model.predict([x_test[:, :, :latent_dim], x_test[:, :, latent_dim:]]).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def auc(y_test, pred):\n",
    "    fpr, tpr, thresholds = metrics.roc_curve(y_test, pred, pos_label=1)\n",
    "    return metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:38: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"in..., outputs=Tensor(\"gl...)`\n"
     ]
    }
   ],
   "source": [
    "avg = AVGPolling(max_len, emb_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_cosine_sim(x, y):\n",
    "    tmp = []\n",
    "    for i,j in zip(x,y):\n",
    "        tmp.append(cosine_similarity(i.reshape(1, -1),j.reshape(1, -1)))\n",
    "    return np.array(tmp).flatten()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vae_pred = matrix_cosine(enc_q_qq, enc_d_qq)\n",
    "w2v_pred = matrix_cosine(avg.model.predict(q_qq), avg.model.predict(d_qq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 35s, sys: 64 ms, total: 2min 35s\n",
      "Wall time: 2min 35s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from l3wtransformer import L3wTransformer\n",
    "l3wt = L3wTransformer()\n",
    "l3wt.fit_on_texts(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 49s, sys: 1 s, total: 1min 50s\n",
      "Wall time: 1min 49s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "max_len = 10\n",
    "# Query-Document Log\n",
    "# q_train = pad_sequences(l3wt.texts_to_sequences(df.q.tolist()), maxlen=max_len)\n",
    "# d_train = pad_sequences(l3wt.texts_to_sequences(df.d.tolist()), maxlen=max_len)\n",
    "\n",
    "# Query-Document Testing data\n",
    "# q_qd_test = pad_sequences(l3wt.texts_to_sequences(df_qd.q.tolist()), maxlen=max_len)\n",
    "# d_qd_test = pad_sequences(l3wt.texts_to_sequences(df_qd.d.tolist()), maxlen=max_len)\n",
    "\n",
    "# Query-Query Testing data\n",
    "q1_qq_test = pad_sequences(l3wt.texts_to_sequences(df_qq.q.tolist()), maxlen=max_len)\n",
    "q2_qq_test = pad_sequences(l3wt.texts_to_sequences(df_qq.d.tolist()), maxlen=max_len)\n",
    "\n",
    "# Query-Query Click data from Minh\n",
    "q1_qq_click_train = pad_sequences(l3wt.texts_to_sequences(q1_qq_click_raw), maxlen=max_len)\n",
    "q2_qq_click_train = pad_sequences(l3wt.texts_to_sequences(q2_qq_click_raw), maxlen=max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.max([len(i) for i in l3wt.texts_to_sequences(df_qq.q.tolist())])\n",
    "max_len = 10\n",
    "# Query-Query data\n",
    "q1_qq_test = pad_sequences(l3wt.texts_to_sequences(df_qq.q.tolist()), maxlen=max_len)\n",
    "q2_qq_test = pad_sequences(l3wt.texts_to_sequences(df_qq.d.tolist()), maxlen=max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class LSTM_Model():\n",
    "    def __init__(self, max_len=10, emb_dim=100, nb_words=50000):\n",
    "\n",
    "        q_input = Input(shape=(max_len,))\n",
    "        d_input = Input(shape=(max_len,))\n",
    "        \n",
    "        emb = Embedding(nb_words, emb_dim, mask_zero=True)\n",
    "\n",
    "        lstm = LSTM(256)\n",
    "\n",
    "        self.q_embed = lstm(emb(q_input))\n",
    "        self.d_embed = lstm(emb(d_input))\n",
    "\n",
    "        concat = Concatenate()([self.q_embed, self.d_embed])\n",
    "\n",
    "        pred = Dense(1, activation='sigmoid')(concat)\n",
    "\n",
    "        self.model = Model(inputs=[q_input, d_input], outputs=pred)\n",
    "        self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "lstm_model = LSTM_Model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MLP():\n",
    "    def __init__(self, input_dim):\n",
    "\n",
    "        que_input = Input(shape=(input_dim,))\n",
    "        doc_input = Input(shape=(input_dim,))\n",
    "        \n",
    "        concat = Concatenate()([que_input, doc_input])\n",
    "\n",
    "        pred = Dense(1, activation='sigmoid')(concat)\n",
    "\n",
    "        self.model = Model(input=[que_input, doc_input], output=pred)\n",
    "        self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Require Negative Sample, otherwise model only predicts 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:9: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "/home/t-jamano/.local/lib/python3.6/site-packages/keras/legacy/layers.py:458: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  name=name)\n",
      "/usr/local/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:14: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=Tensor(\"de...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      " - 401s - loss: 0.6932 - acc: 0.4997\n",
      "0.509195223376\n",
      "Epoch 1/1\n",
      " - 401s - loss: 0.6932 - acc: 0.4991\n",
      "0.474715915408\n",
      "Epoch 1/1\n",
      " - 403s - loss: 0.6932 - acc: 0.4986\n",
      "0.513509306411\n"
     ]
    }
   ],
   "source": [
    "mlp_model = MLP(max_len)\n",
    "\n",
    "q2_neg_qq_click_train = np.copy(q2_qq_click_train)\n",
    "# np.random.shuffle(q2_neg_qq_click_train)\n",
    "lstm_model = LSTM_Model()\n",
    "# x_train = [np.concatenate((q1_qq_click_train, q1_qq_click_train)), np.concatenate((q2_qq_click_train, q2_neg_qq_click_train))] \n",
    "y_train = np.array([1] * len(df_qq_click) + [0] * len(df_qq_click))\n",
    "for i in range(3):\n",
    "    \n",
    "#     np.random.shuffle(q2_neg_qq_click_train)\n",
    "#     x_train = [np.concatenate((q1_qq_click_train, q1_qq_click_train)), np.concatenate((q2_qq_click_train, q2_neg_qq_click_train))] \n",
    "\n",
    "    lstm_model.model.fit(x_train, y_train, verbose=2, batch_size=256)\n",
    "\n",
    "    pred = lstm_model.model.predict([q1_qq_test, q2_qq_test])\n",
    "    print(auc(df_qq.label.values, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# y_test = [0 if i == \"Bad\" else 1 for i in df_qd.label.tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "max_len = 10\n",
    "emb_dim = 50\n",
    "NB_WORDS = 50000\n",
    "latent_dim = 100\n",
    "intermediate_dim = 256\n",
    "epsilon_std = 1.0\n",
    "num_sampled=500\n",
    "act = ELU()\n",
    "vae, encoder, decoder = create_lstm_vae(emb_dim, max_len, batch_size, intermediate_dim, latent_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# http://alexadam.ca/ml/2017/05/05/keras-vae.html\n",
    "class EMB_LSTM_VAE():\n",
    "    def __init__(self, vocab_size=50000, max_length=10, latent_rep_size=50):\n",
    "        self.encoder = None\n",
    "        self.decoder = None\n",
    "        self.sentiment_predictor = None\n",
    "        self.autoencoder = None\n",
    "        \n",
    "        self.max_length = max_length\n",
    "        self.vocab_size = vocab_size\n",
    "        self.latent_rep_size = latent_rep_size\n",
    "\n",
    "        x = Input(shape=(max_length,))\n",
    "        self.x_embed = Embedding(vocab_size, 100, input_length=max_length, mask_zero=True)(x)\n",
    "\n",
    "        vae_loss, encoded = self._build_encoder(self.x_embed, latent_rep_size=latent_rep_size, max_length=max_length)\n",
    "        self.encoder = Model(x, encoded)\n",
    "\n",
    "        encoded_input = Input(shape=(latent_rep_size,))\n",
    "#         predicted_sentiment = self._build_sentiment_predictor(encoded_input)\n",
    "#         self.sentiment_predictor = Model(encoded_input, predicted_sentiment)\n",
    "\n",
    "        decoded = self._build_decoder(encoded_input, vocab_size, max_length)\n",
    "        self.decoder = Model(encoded_input, decoded)\n",
    "\n",
    "#         self.autoencoder = Model(x, [self._build_decoder(encoded, vocab_size, max_length), self._build_sentiment_predictor(encoded)])\n",
    "        self.model = Model(x, self._build_decoder(encoded, vocab_size, max_length))\n",
    "\n",
    "        self.model.compile(optimizer='Adam',\n",
    "                                 loss=vae_loss)\n",
    "        \n",
    "    def _build_encoder(self, x, latent_rep_size=200, max_length=300, epsilon_std=0.01):\n",
    "#         h = Bidirectional(LSTM(500, return_sequences=True, name='lstm_1'), merge_mode='concat')(x)\n",
    "#         h = Bidirectional(LSTM(500, return_sequences=False, name='lstm_2'), merge_mode='concat')(h)\n",
    "#         h = Dense(435, activation='relu', name='dense_1')(h)\n",
    "        h = LSTM(500, name='lstm_1')(x)\n",
    "\n",
    "\n",
    "        def sampling(args):\n",
    "            z_mean_, z_log_var_ = args\n",
    "            batch_size = K.shape(z_mean_)[0]\n",
    "            epsilon = K.random_normal(shape=(batch_size, latent_rep_size), mean=0., stddev=epsilon_std)\n",
    "            return z_mean_ + K.exp(z_log_var_ / 2) * epsilon\n",
    "\n",
    "        z_mean = Dense(latent_rep_size, name='z_mean', activation='linear')(h)\n",
    "        z_log_var = Dense(latent_rep_size, name='z_log_var', activation='linear')(h)\n",
    "\n",
    "        def vae_loss(x1, x_decoded_mean):\n",
    "            x1 = K.flatten(self.x_embed)\n",
    "            x_decoded_mean = K.flatten(x_decoded_mean)\n",
    "            xent_loss = max_length * objectives.cosine(x1, x_decoded_mean)\n",
    "            kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)\n",
    "            return xent_loss + kl_loss\n",
    "\n",
    "        return (vae_loss, Lambda(sampling, output_shape=(latent_rep_size,), name='lambda')([z_mean, z_log_var]))\n",
    "    \n",
    "    def _build_decoder(self, encoded, vocab_size, max_length):\n",
    "        repeated_context = RepeatVector(max_length)(encoded)\n",
    "\n",
    "#         h = LSTM(500, return_sequences=True, name='dec_lstm_1')(repeated_context)\n",
    "#         h = LSTM(500, return_sequences=True, name='dec_lstm_2')(h)\n",
    "        h = LSTM(500, return_sequences=True, name='dec_lstm_1')(repeated_context)\n",
    "#         h = LSTM(500, return_sequences=True, name='dec_lstm_2')(h)\n",
    "#         decoded = h\n",
    "        decoded = Flatten()(TimeDistributed(Dense(100, activation='linear'), name='decoded_mean')(h))\n",
    "        return decoded\n",
    "    \n",
    "    def _build_sentiment_predictor(self, encoded):\n",
    "        h = Dense(100, activation='linear')(encoded)\n",
    "\n",
    "        return Dense(1, activation='sigmoid', name='pred')(h)\n",
    "\n",
    "emb_lstm_vae = EMB_LSTM_VAE()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 10)"
      ]
     },
     "execution_count": 375,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_lstm_vae.model.predict(np.random.randint(50000, size=(10,10))).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      " - 165s - loss: -9.7349e+00\n",
      "0.507015944317\n",
      "Epoch 1/1\n",
      " - 150s - loss: -9.9135e+00\n",
      "0.533167293024\n",
      "Epoch 1/1\n",
      " - 151s - loss: -9.9363e+00\n",
      "0.522159710022\n",
      "Epoch 1/1\n",
      " - 152s - loss: -9.9853e+00\n",
      "0.542862861082\n",
      "Epoch 1/1\n",
      " - 149s - loss: -9.9944e+00\n",
      "0.524605839578\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    emb_lstm_vae.model.fit(q1_qq_click_train, q1_qq_click_train, verbose=2, batch_size=batch_size)\n",
    "    pred = get_cosine_sim(emb_lstm_vae.encoder.predict(q1_qq_test), emb_lstm_vae.encoder.predict(q2_qq_test))\n",
    "    print(auc(df_qq.label.values, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.52507282794814203"
      ]
     },
     "execution_count": 391,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = get_cosine_sim(q1_qq_test, q2_qq_test)\n",
    "auc(df_qq.label.values, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      " - 148s - loss: -9.9773e+00\n",
      "0.520213925148\n",
      "Epoch 1/1\n",
      " - 149s - loss: -9.9963e+00\n",
      "0.516400186795\n",
      "Epoch 1/1\n",
      " - 148s - loss: -9.9737e+00\n",
      "0.530832351175\n",
      "Epoch 1/1\n",
      " - 148s - loss: -9.9949e+00\n",
      "0.523783050546\n",
      "Epoch 1/1\n",
      " - 149s - loss: -9.9970e+00\n",
      "0.516822700082\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    emb_lstm_vae.model.fit(q2_qq_click_train, q2_qq_click_train, verbose=2, batch_size=batch_size)\n",
    "    pred = get_cosine_sim(emb_lstm_vae.encoder.predict(q1_qq_test), emb_lstm_vae.encoder.predict(q2_qq_test))\n",
    "    print(auc(df_qq.label.values, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}