NLP/WordEmbedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = ['king is a strong man',\n",
    "          'queen is a wise woman',\n",
    "          'boy is a young man',\n",
    "          'girl is a young woman',\n",
    "          'prince is a young king',\n",
    "          'princess is a young queen',\n",
    "          'man is strong',\n",
    "          'woman is pretty',\n",
    "          'prince is a boy will be king',\n",
    "          'princess is a girl will be queen']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "去除一些停止词："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['king strong man', 'queen wise woman', 'boy young man', 'girl young woman', 'prince young king', 'princess young queen', 'man strong', 'woman pretty', 'prince boy king', 'princess girl queen']\n"
     ]
    }
   ],
   "source": [
    "def remove_stop_words(corpus):\n",
    "    stop_words = set(['is', 'a', 'will', 'be'])\n",
    "    return [\" \".join([word for word in text.split() if word not in stop_words]) for text in corpus]\n",
    "\n",
    "\n",
    "corpus = remove_stop_words(corpus)\n",
    "print(corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "找出词语的集合："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_set = set()\n",
    "for text in corpus:\n",
    "    for word in text.split(' '):\n",
    "        words_set.add(word)\n",
    "\n",
    "# words_set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "维护一个word-number的映射表，使用字典实现："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2int=dict()\n",
    "int2word=dict()\n",
    "for idx,word in enumerate(words_set):\n",
    "    word2int[word]=idx\n",
    "    int2word[idx]=word\n",
    "\n",
    "# word2int"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用滑动窗口生成数据："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "WINDOW_SIZE = 2    # 中心词与左右边界的距离\n",
    "center_words = list()\n",
    "context_words = list()\n",
    "\n",
    "for text in corpus:\n",
    "    words = text.split()\n",
    "    for idx, center_word in enumerate(words):\n",
    "        # 限定区域\n",
    "        for context_word in words[max(0, idx-WINDOW_SIZE):min(idx+WINDOW_SIZE, len(words))+1]:\n",
    "            if context_word != center_word:\n",
    "                center_words.append(center_word)\n",
    "                context_words.append(context_word)\n",
    "                \n",
    "print(center_words,context_words,sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将数据转换成one-hot向量："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ONE_HOT_DIM = len(words_set)\n",
    "\n",
    "def to_onehot(word):\n",
    "    res=np.zeros(ONE_HOT_DIM)\n",
    "    res[word2int[word]]=1\n",
    "    return res\n",
    "\n",
    "X_train=np.array(list(map(to_onehot,center_words)))\n",
    "Y_train=np.array(list(map(to_onehot,context_words)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 网络设计\n",
    "最简单的词嵌入模型是一个单隐层多输出的神经网络，输入层的单元数等于词库的大小，隐藏层单元数等于嵌入词向量的维度，输出层单元数仍然等于词库的大小。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "unit_I = len(words_set)\n",
    "unit_h1 = 2    # embedding size\n",
    "unit_O = unit_I"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 搭建网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-9-17902aafffc4>:7: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/losses/losses_impl.py:209: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "# 输入必须是可由用户指定的，所以设为placeholder\n",
    "X = tf.placeholder(tf.float32, [None, unit_I])\n",
    "Y = tf.placeholder(tf.float32, [None, unit_O])\n",
    "\n",
    "# 网络结构图\n",
    "with tf.name_scope('WordEmb'):\n",
    "    hidden1 = tf.layers.dense(X, unit_h1,\n",
    "                              activation=None, name='emb')    # 隐层即嵌入层\n",
    "    # 嵌入层与FC层的区别就在于，做嵌入需要的是参数，而FC层需要的输出\n",
    "    with tf.variable_scope('emb', reuse=True):\n",
    "        w = tf.get_variable(\"kernel\")\n",
    "        b = tf.get_variable(\"bias\")\n",
    "\n",
    "    logits = tf.layers.dense(hidden1, unit_O, activation=None)\n",
    "\n",
    "# 评估子图\n",
    "with tf.name_scope('Eval'):\n",
    "    loss = tf.losses.softmax_cross_entropy(onehot_labels=Y, logits=logits)\n",
    "\n",
    "# 优化图\n",
    "with tf.name_scope('train_op'):\n",
    "    lr = 1e-2\n",
    "    train_op = tf.train.GradientDescentOptimizer(lr).minimize(loss)\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 0 loss is :  2.5260603\n",
      "iteration 2000 loss is :  2.2669108\n",
      "iteration 4000 loss is :  2.0499022\n",
      "iteration 6000 loss is :  1.9456667\n",
      "iteration 8000 loss is :  1.88323\n"
     ]
    }
   ],
   "source": [
    "with tf.Session(config=config) as sess:\n",
    "    sess.run(init)\n",
    "    iteration = 10000\n",
    "    \n",
    "    for i in range(iteration):\n",
    "        sess.run(train_op, feed_dict={X: X_train, Y: Y_train})\n",
    "        if i % 2000 == 0:\n",
    "            print('iteration '+str(i)+' loss is : ', sess.run(loss, feed_dict={X: X_train, Y: Y_train}))\n",
    "            \n",
    "    embs=sess.run(w+b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAH89JREFUeJzt3XtwVdXd//H3SrjkJ5fAQLygkZNOIQZyEnIS0DSEW4pERSmUKGlwCBGpoPWxo5E68oAXOuMIotUiVh9LKspFoFClVkMA5WIquZBACASoxtTqtKFPQAKCCVm/P4DzSAkXyUl2svm8ZjKTfc7aa3/XGfNxs87aextrLSIi4h5BThcgIiKBpWAXEXEZBbuIiMso2EVEXEbBLiLiMgp2ERGXUbCLiLiMgl1ExGUU7CIiLtPOiYP27NnTejweJw4tItJmFRUVHbDWhl2onSPB7vF4KCwsdOLQIiJtljHm84tpp6kYERGXUbCLiLiMgl1ExGUU7CIiLqNgFxFxGQW7iIjLKNhFRFxGwS4i4jJNDnZjTIgxZpsxptQYs8sY82QgChMRkUsTiCtPjwMjrLW1xpj2wBZjzF+stX8NQN8iIvI9NTnYrbUWqD212f7Uj21qvyIicmkCMsdujAk2xpQA/wLWWWs/CUS/IiLy/QUk2K21J6y1A4DrgEHGmOj/bGOMmWqMKTTGFFZXVwfisCIi0oiAroqx1h4ENgKpjbz3qrU2wVqbEBZ2wbtOiojIJQrEqpgwY0y3U7//P2AksKep/YqIyKUJxKqYa4A/GGOCOfk/irettWsD0K+IiFyCQKyK2QHEBaAWEREJAF15KiLiMgp2ERGXUbCLiLiMgl1ExGUU7CIiLqNgFxFxGQW7iIjLKNhFRFxGwS4i4jIKdhERl1GwS6vzwgsvcPToUafLEGmzFOzS6pwv2E+cONHC1Yi0PQp2cdSRI0e47bbbiI2NJTo6mieffJIvv/yS4cOHM3z4cAA6d+7Mww8/TGxsLPn5+axfv564uDi8Xi9ZWVkcP34cAI/Hw+zZs/H5fHi9XvbsOXn36OrqakaOHEn//v2ZMmUKvXv35sCBA46NWaS5KdjFUe+//z69evWitLSUsrIyHnroIXr16sXGjRvZuHEjcDL8b7zxRkpLS0lISCAzM5Ply5ezc+dO6uvrWbhwob+/nj17UlxczLRp05g3bx4ATz75JCNGjGDXrl2MHz+eqqoqR8Yq0lIU7OIor9fLunXrmDFjBps3byY0NPSsNsHBwfz0pz8FoKKigoiICPr27QvApEmT2LRpk7/tuHHjAIiPj6eyshKALVu2MGHCBABSU1Pp3r17cw5JxHGBeNCGyCXr27cvxcXFvPfee8ycOZOUlJSz2oSEhBAcHHxR/XXs2BE4+T+D+vr6gNYq0lbojF0c9eWXX3LFFVcwceJEsrOzKS4upkuXLhw+fLjR9pGRkVRWVrJ//34AFi9ezNChQ897jKSkJN5++20AcnNzqampCewgRFoZnbGLo3bu3El2djZBQUG0b9+ehQsXkp+fT2pqqn+u/btCQkJYtGgRaWlp1NfXM3DgQO67777zHmP27Nmkp6ezePFiEhMTufrqq+nSpUtzDkvEUcZa2+IHTUhIsIWFhS1+XLk8HT9+nODgYNq1a0d+fj7Tpk2jpKTE6bJEvjdjTJG1NuFC7XTGLq5XVVXFnXfeSUNDAx06dOC1115zuiSRZqVgF9fr06cP27dvd7oMkRajL09FRFxGwS4i4jIKdhERl1Gwi4i4jIJdRMRlFOwiIi7T5GA3xoQbYzYaY8qNMbuMMf8ViMJEROTSBGIdez3wsLW22BjTBSgyxqyz1pYHoG8REfmemnzGbq39ylpbfOr3w8Bu4Nqm9isiIpcmoHPsxhgPEAd8Esh+RUTk4gUs2I0xnYFVwEPW2q8beX+qMabQGFNYXV0dqMOKiMh/CEiwG2PaczLU37LW/rGxNtbaV621CdbahLCwsEAcVkREGhGIVTEGeB3Yba2d3/SSRESkKQJxxp4E3A2MMMaUnPq5NQD9iojIJWjyckdr7RbABKAWEREJAF15KiLiMgp2ERGXUbCLiLiMgl1ExGUU7CIiLqNgFxFxGQW7tGmVlZVER0c7XYZIq6JgFxFxGQW7tHn19fVkZGQQFRXF+PHjOXr0KOvXrycuLg6v10tWVhbHjx9nw4YN/OQnP/Hvt27dOsaOHetg5SLNQ8EubV5FRQXTp09n9+7ddO3alfnz55OZmcny5cvZuXMn9fX1LFy4kOHDh7Nnzx5O31100aJFZGVlOVy9SOAp2KXNCw8PJykpCYCJEyeyfv16IiIi6Nu3LwCTJk1i06ZNGGO4++67efPNNzl48CD5+fnccsstTpYu0iwC8Wg8EUedvMHo/+nWrRv//ve/G207efJkbr/9dkJCQkhLS6NdO/0JiPvojF3avKqqKvLz8wFYsmQJCQkJVFZWsn//fgAWL17M0KFDAejVqxe9evVizpw5TJ482bGaRZqTgl3avMjISBYsWEBUVBQ1NTX88pe/ZNGiRaSlpeH1egkKCuK+++7zt8/IyCA8PJyoqCgHqxZpPvp3qLRpHo+HPXv2nPV6SkoK27dvb3SfLVu2cO+99zZ3aSKOUbDLZSU+Pp5OnTrx3HPPOV2KSLPRVIwAjV/BWVhYyIMPPuhQRc2jqKiITZs20bFjR6dLEWk2OmOXc0pISCAhIcHpMkTke9IZu5zl008/JS4ujrlz5zJ69GgAnnjiCbKyshg2bBg/+MEPePHFF/3tn376aSIjIxk8eDDp6enMmzfPqdJFBJ2xy3+oqKhgwoQJ5OTkUFNTw0cffeR/b8+ePWzcuJHDhw8TGRnJtGnTKCkpYdWqVZSWllJXV4fP5yM+Pt7BEYiIztjFr7q6mjFjxvDWW28RGxt71vu33XYbHTt2pGfPnlx55ZX885//ZOvWrYwZM4aQkBC6dOnC7bff7kDlIvJdCnbxCw0N5frrr2fLli2Nvv/dLxyDg4Opr69vqdJE5HtQsItfhw4dWL16NW+88QZLliy5qH2SkpJ49913OXbsGLW1taxdu7aZqxSRC1Gwyxk6derE2rVref755/n6668v2H7gwIHccccdxMTEcMstt+D1egkNDW2BSkXkXIy1tsUPmpCQYAsLC1v8uNI8amtr6dy5M0ePHmXIkCG8+uqr+Hw+p8sScR1jTJG19oJrkLUqRpps6tSplJeXc+zYMSZNmqRQF3GYgl2a7GLn40WkZWiOXUTEZQIS7MaY3xtj/mWMKQtEfyIicukCdcaeA6QGqC8REWmCgAS7tXYT8L+B6EtERJqmxebYjTFTjTGFxpjC00+Jl8CorKzkhhtuIDMzk759+5KRkUFeXh5JSUn06dOHbdu2sW3bNhITE4mLi+NHP/oRFRUVAOTk5DBu3DhSU1Pp06cPjz76qMOjEZEms9YG5AfwAGUX0zY+Pt5K4Hz22Wc2ODjY7tixw544ccL6fD47efJk29DQYNesWWPHjBljDx06ZOvq6qy11q5bt86OGzfOWmvtokWLbEREhD148KD95ptv7PXXX2+rqqqcHI6InANQaC8iY7Xc0SUiIiLwer0A9O/fn5SUFIwxeL1eKisrOXToEJMmTWLfvn0YY6irq/Pvm5KS4r9atF+/fnz++eeEh4c7Mg4RaTotd3SJ796gKygoyL8dFBREfX09//3f/83w4cMpKyvz39ulsX11cy+Rti9Qyx2XAvlApDHmC2PMPYHoVwLn0KFDXHvttcDJeXURca9ArYpJt9ZeY61tb629zlr7eiD6lcbNmjWLvLy877XPo48+ymOPPUZcXJzOyEVcTjcBa2NOnDhBcHCw02WIiAMu9iZgmmNvRU4vW8zIyCAqKorx48dz9OhRPB4PM2bMwOfzsWLFCjIzM1m5ciUAHo+H2bNn4/P58Hq97NmzBzh5x8XJkyfj9XqJiYlh1apVAOTm5pKYmIjP5yMtLY3a2lrHxisizUPB3spUVFQwffp0du/eTdeuXXn55ZcB6NGjB8XFxUyYMOGsfXr27ElxcTHTpk3zP0j66aefJjQ0lJ07d7Jjxw5GjBjBgQMHmDNnDnl5eRQXF5OQkMD8+fNbdHwi0vy03LGVCQ8PJykpCYCJEyfy4osvAnDXXXedc59x48YBEB8fzx//+EcA8vLyWLZsmb9N9+7dWbt2LeXl5f7+v/32WxITE5tlHCLiHAV7K2OMaXS7U6dO59zn9HLFCy1VtNYycuRIli5dGoBKRaS10lRMK1NVVUV+fj5w8j7ngwcPvqR+Ro4cyYIFC/zbNTU13HTTTWzdupX9+/cDcOTIEfbu3dv0okWkVVGwtzKRkZEsWLCAqKgoampqmDZt2iX1M3PmTGpqaoiOjiY2NpaNGzcSFhZGTk4O6enpxMTEkJiY6P+yVUTcQ8sdW5HKykpGjx5NWZluay8iZ9NyRxGRy5SCvRXxeDw6WxeRJlOwi4i4jIJdRMRlFOwiIi6jYBcRcRkFu4iIyyjYRURcRsEuIuIyCnYREZdRsIuIuIyCXUTEZRTsIiIuo2AXEXEZBbu0mFmzZvHCCy/4tx9//HF+85vfkJ2dTXR0NF6vl+XLlwPw4YcfMnr0aH/bBx54gJycHODcD/Curq5m5MiR9O/fnylTptC7d28OHDjQcgMUaSUU7NJisrKyeOONNwBoaGhg2bJlXHfddZSUlFBaWkpeXh7Z2dl89dVXF+yrsQd4P/nkk4wYMYJdu3Yxfvx4qqqqmnU80nrMmjWLvLw8p8toNfTMU2kxHo+HHj16sH37dv75z38SFxfHli1bSE9PJzg4mKuuuoqhQ4dSUFBA165dz9tXYw/w3rJlC6tXrwYgNTWV7t27N++ApFU4ceIETz31lNNltCo6Y5cWNWXKFHJycli0aBFZWVnnbNeuXTsaGhr828eOHTvj/Yt9gLe0bZWVldxwww1kZGQQFRXF+PHjOXr0KB6PhxkzZuDz+VixYgWZmZmsXLkSOPdUXW1tLZMnT8br9RITE8OqVasAyM3NJTExEZ/PR1paGrW1tQD86le/ol+/fsTExPDII48AsGLFCv/jJocMGeLAJ3JxFOzSosaOHcv7779PQUEBo0aNIjk5meXLl3PixAmqq6vZtGkTgwYNonfv3pSXl3P8+HEOHjzI+vXrL9h3UlISb7/9NnDyj7Wmpqa5hyMtoKKigunTp7N79266du3Kyy+/DECPHj0oLi5mwoQJZ+3T2FTd008/TWhoKDt37mTHjh2MGDGCAwcOMGfOHPLy8iguLiYhIYH58+fz73//m9WrV7Nr1y527NjBzJkzAXjqqaf44IMPKC0t5Z133mm5D+F7CshUjDEmFfgNEAz8j7X2mUD0K+7ToUMHhg8fTrdu3QgODmbs2LHk5+cTGxuLMYZnn32Wq6++GoA777yT6OhoIiIiiIuLu2Dfs2fPJj09ncWLF5OYmMjVV19Nly5dmntI0szCw8NJSkoCYOLEibz44osA3HXXXefcp7Gpury8PJYtW+Zv0717d9auXUt5ebm//2+//ZbExERCQ0MJCQnhnnvuYfTo0f4v8pOSksjMzOTOO+/0H6M1anKwG2OCgQXASOALoMAY8461trypfYv7NDQ08Ne//pUVK1YAYIxh7ty5zJ0796y2zz77LM8+++xZr1dWVvp/T0hI4MMPPwQgNDSUDz74gHbt2pGfn09BQYF/ykbaLmNMo9udOnU65z4XO1VnrWXkyJEsXbr0rPe2bdvG+vXrWblyJb/97W/ZsGEDr7zyCp988gl//vOfiY+Pp6ioiB49elzKsJpVIKZiBgH7rbWfWmu/BZYBYwLQr7hMeXk5P/zhD0lJSaFPnz4B77+qqoqBAwcSGxvLgw8+yGuvvRbwY0jLq6qqIj8/H4AlS5YwePDgS+pn5MiRLFiwwL9dU1PDTTfdxNatW9m/fz8AR44cYe/evdTW1nLo0CFuvfVWnn/+eUpLSwH429/+xo033shTTz1FWFgYf//735s4uuYRiKmYa4Hvju4L4MYA9Csu069fPz799NNm679Pnz5s37692foXZ0RGRrJgwQKysrLo168f06ZN46WXXvre/cycOZP777+f6OhogoODmT17NuPGjSMnJ4f09HSOHz8OwJw5c+jSpQtjxozh2LFjWGuZP38+ANnZ2ezbtw9rLSkpKcTGxgZ0rIFirLVN68CY8UCqtXbKqe27gRuttQ/8R7upwFSA66+/Pv7zzz9v0nFFxP0qKysZPXo0ZWVlTpfSKhhjiqy1CRdqF4ipmH8A4d/Zvu7Ua2ew1r5qrU2w1iaEhYUF4LAiItKYQAR7AdDHGBNhjOkATABa7zogEWkzPB6PztYvQZPn2K219caYB4APOLnc8ffW2l1NrkxERC5JQNaxW2vfA94LRF8iItI0uvJURMRlFOwiIi6jYBcRcRkFu4iIyyjYRURcRsEuIuIyCnYREZdRsIuIuIyCXUTEZRTsIiIuo2AXEXEZBbuIiMso2EVEXEbBLiLiMgp2ERGXUbCLtIC5c+fy4osvAvDLX/6SESNGALBhwwYyMjJYunQpXq+X6OhoZsyY4d+vc+fOZGdn079/f3784x+zbds2hg0bxg9+8APeeefkg8oqKytJTk7G5/Ph8/n4+OOPAfjwww8ZNmwY48eP54YbbiAjI4OmPuNY2gYFu0gLSE5OZvPmzQAUFhZSW1tLXV0dmzdvpm/fvsyYMYMNGzZQUlJCQUEBa9asAeDIkSOMGDGCXbt20aVLF2bOnMm6detYvXo1s2bNAuDKK69k3bp1FBcXs3z5ch588EH/cbdv384LL7xAeXk5n376KVu3bm35wUuLU7CLtID4+HiKior4+uuv6dixI4mJiRQWFrJ582a6devGsGHDCAsLo127dmRkZLBp0yYAOnToQGpqKgBer5ehQ4fSvn17vF4vlZWVANTV1XHvvffi9XpJS0ujvLzcf9xBgwZx3XXXERQUxIABA/z7iLsF5NF4InJ+7du3JyIigpycHH70ox8RExPDxo0b2b9/Px6Ph6KionPuZ4wBICgoiI4dO/p/r6+vB+D555/nqquuorS0lIaGBkJCQvz7n24PEBwc7N9H3E1n7CItJDk5mXnz5jFkyBCSk5N55ZVXiIuLY9CgQXz00UccOHCAEydOsHTpUoYOHXrR/R46dIhrrrmGoKAgFi9ezIkTJ5pxFNIWKNjFVW699VYOHjzodBmNSk5O5quvviIxMZGrrrqKkJAQkpOTueaaa3jmmWcYPnw4sbGxxMfHM2bMmIvud/r06fzhD38gNjaWPXv20KlTp2YchbQFxolvyRMSEmxhYWGLH1dEpC0zxhRZaxMu1E5n7NKmXGjZoMfj4cCBAxw5coTbbruN2NhYoqOjWb58OQBFRUUMHTqU+Ph4Ro0axVdffeXYWESai4Jd2pTzLRscMmSIv937779Pr169KC0tpaysjNTUVOrq6vjFL37BypUrKSoqIisri8cff9ypoYg0GwW7tCnnWzaYnJzsb+f1elm3bh0zZsxg8+bNhIaGUlFRQVlZGSNHjmTAgAHMmTOHL774IiB1zZo1i7y8vEbfy8zMZOXKlQE5jsjF0HJHaVPOt2wwKirK365v374UFxfz3nvvMXPmTFJSUhg7diz9+/cnPz8/4HU99dRTjb6uFSriBJ2xS5tzrmWDp9d7A3z55ZdcccUVTJw4kezsbIqLi4mMjKS6utof7HV1dezatet7H//pp58mMjKSwYMHk56ezrx58844K/d4PMyYMQOfz8eKFSsCM2iR76FJZ+zGmDTgCSAKGGSt1VIXaXbJycn8+te/JjExkU6dOvmXDX7Xzp07yc7OJigoiPbt27Nw4UI6dOjAypUrefDBBzl06BD19fU89NBD9O/f/6KPXVBQwKpVqygtLaWurg6fz0d8fPxZ7Xr06EFxcTFwcr5fpCU1dSqmDBgH/C4AtYhclJSUFOrq6vzbe/fu9f9++pL5UaNGMWrUqLP2HTBggP9y/UuxdetWxowZQ0hICCEhIdx+++2Ntrvrrrsu+RgiTdWkYLfW7gbO+CewiKCLhMRRmmOXS7ZmzZozbjiVk5PDl19+6WBFzS8pKYl3332XY8eOUVtby9q1a50uSeQsFwx2Y0yeMaaskZ+Lv+b5ZD9TjTGFxpjC6urqS69YWtT5VnVcjsE+cOBA7rjjDmJiYrjlllvwer2EhoY6XZbImay1Tf4BPgQSLrZ9fHy8Fed99tlnNjIy0v7sZz+zN9xwg/3pT39qjxw5Ynv37m0fffRRGxcXZ5cuXWr3799vR40aZX0+nx08eLDdvXu33bp1q+3evbv1eDw2NjbWPvPMM7ZTp062b9++NjY21q5du9aOGTPGf6zc3Fz7k5/8xMHRBs7hw4ettdYeOXLExsfH26KiIocrkssFUGgvImO1jv0yV1FRweuvv05SUhJZWVm8/PLLwJmrOlJSUnjllVfo06cPn3zyCdOnT2fDhg3ccccdjB49mvHjxwPwl7/8hXnz5pGQkIC1locffpjq6mrCwsJYtGgRWVlZjo0zkKZOnUp5eTnHjh1j0qRJ+Hw+p0sSOUNTlzuOBV4CwoA/G2NKrLVnL0WQVis8PJykpCQAJk6c6L8Py+lVHbW1tXz88cekpaX59zl+/PgF+zXGcPfdd/Pmm28yefJk8vPzeeONN5phBC1vyZIlTpcgcl5NXRWzGlgdoFrEAf+5oun09ulVHQ0NDXTr1o2SkpLv3ffkyZO5/fbbCQkJIS0tjXbt9A9EkZagVTGXuaqqKv+VmEuWLGHw4MFnvN+1a1ciIiL8V1BaayktLQWgS5cuHD582N/2P7d79epFr169mDNnDpMnT27uoYjIKQr2y1xkZCQLFiwgKiqKmpoapk2bdlabt956i9dff53Y2Fj69+/Pn/70JwAmTJjA3LlziYuL429/+xuZmZncd999DBgwgG+++QaAjIwMwsPDz7iPi4g0Lz1o4zJWWVnJ6NGjKSsra7ZjPPDAA8TFxXHPPfc02zFELhcX+6ANTXpKs4mPj6dTp04899xzTpcicllRsF/GPB5Ps56tFxUVNVvfInJummMXEXEZBbuIiMso2EVEXEbBLiLiMgp2ERGXUbCLiLiMgl1ExGUU7CKt1K9//Wv69u3L4MGDSU9PZ968eQwbNozTV20fOHAAj8cDnHwgSnZ2NgMHDiQmJobf/e7/HkM8d+5c/+uzZ88GTl51HBUVxb333kv//v25+eab/beBkLZPwS7SChUVFbFs2TJKSkp47733KCgoOG/7119/ndDQUAoKCigoKOC1117js88+Izc3l3379rFt2zZKSkooKiryP8x737593H///ezatYtu3bqxatWqlhiatABdeSrSCm3evJmxY8dyxRVXAHDHHXect31ubi47duxg5cqVABw6dIh9+/aRm5tLbm4ucXFxwMn76+/bt4/rr7+eiIgIBgwYAJy8/UNlZWXzDUhalIJdpA1p164dDQ0NABw7dsz/urWWl156iVGjznzOzQcffMBjjz3Gz3/+8zNer6yspGPHjv7t4OBgTcW4iKZiRFqhIUOGsGbNGr755hsOHz7Mu+++C5y8v8/pe/CcPjsHGDVqFAsXLqSurg6AvXv3cuTIEUaNGsXvf/97amtrAfjHP/7Bv/71rxYejbQ0nbGLtEI+n4+77rqL2NhYrrzySgYOHAjAI488wp133smrr77Kbbfd5m8/ZcoUKisr8fl8WGsJCwtjzZo13HzzzezevZvExEQAOnfuzJtvvklwcLAj45KWofuxi7QBTzzxBJ07d+aRRx5xuhRx0MXej11TMSIiLqOpGJE24IknnnC6BGlDdMYuIuIyCnYREZdRsIuIuIyCXUTEZRTsIiIuo2AXEXEZBbuIiMs0KdiNMXONMXuMMTuMMauNMd0CVZiIiFyapp6xrwOirbUxwF7gsaaXJCIiTdGkYLfW5lpr609t/hW4rukliYhIUwRyjj0L+Mu53jTGTDXGFBpjCqurqwN4WBER+a4L3ivGGJMHXN3IW49ba/90qs3jQD3w1rn6sda+CrwKJ+/ueEnViojIBV0w2K21Pz7f+8aYTGA0kGKduAewiIicoUl3dzTGpAKPAkOttUcDU5KIiDRFU+fYfwt0AdYZY0qMMa8EoCYREWmCJp2xW2t/GKhCREQkMHTlqYiIyyjYRURcRsEuIuIyCnYREZdRsIuIuIyCXUTEZRTsIiIuo2AXEXEZBbuIiMso2EVEXEbBLiLiMgp2ERGXUbCLiLiMgl1ExGUU7CIiLqNgFxFxGQW7iIjLGCeeP22MqQY+b/EDtw49gQNOF9HK6DM5mz6Ts+kzgd7W2rALNXIk2C9nxphCa22C03W0JvpMzqbP5Gz6TC6epmJERFxGwS4i4jIK9pb3qtMFtEL6TM6mz+Rs+kwukubYRURcRmfsIiIuo2B3gDEmzRizyxjTYIy5bL/lN8akGmMqjDH7jTG/crqe1sAY83tjzL+MMWVO19IaGGPCjTEbjTHlp/5m/svpmtoCBbszyoBxwCanC3GKMSYYWADcAvQD0o0x/ZytqlXIAVKdLqIVqQcettb2A24C7td/JxemYHeAtXa3tbbC6TocNgjYb6391Fr7LbAMGONwTY6z1m4C/tfpOloLa+1X1triU78fBnYD1zpbVeunYBenXAv8/TvbX6A/WDkPY4wHiAM+cbaS1q+d0wW4lTEmD7i6kbcet9b+qaXrEWnLjDGdgVXAQ9bar52up7VTsDcTa+2Pna6hlfsHEP6d7etOvSZyBmNMe06G+lvW2j86XU9boKkYcUoB0McYE2GM6QBMAN5xuCZpZYwxBngd2G2tne90PW2Fgt0BxpixxpgvgETgz8aYD5yuqaVZa+uBB4APOPmF2NvW2l3OVuU8Y8xSIB+INMZ8YYy5x+maHJYE3A2MMMaUnPq51emiWjtdeSoi4jI6YxcRcRkFu4iIyyjYRURcRsEuIuIyCnYREZdRsIuIuIyCXUTEZRTsIiIu8/8BM+SjWKLlVVIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.clf()\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "for idx,emb in enumerate(embs):\n",
    "    ax.annotate(int2word[idx], emb)\n",
    "\n",
    "PADDING = 1.0\n",
    "x_axis_min = np.amin(embs, axis=0)[0] - PADDING\n",
    "y_axis_min = np.amin(embs, axis=0)[1] - PADDING\n",
    "x_axis_max = np.amax(embs, axis=0)[0] + PADDING\n",
    "y_axis_max = np.amax(embs, axis=0)[1] + PADDING\n",
    "\n",
    "plt.xlim(x_axis_min, x_axis_max)\n",
    "plt.ylim(y_axis_min, y_axis_max)\n",
    "plt.rcParams[\"figure.figsize\"] = (10, 10)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}