01-word-vectors/word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2Vec\n",
    "Word2Vec uses NCE loss function, but it is not yet included in PyTorch. So we will just use softmax. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch as th\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch.utils.data as data_utils\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "word_pair = [['cat', 'white'],\n",
    "             ['cat', 'animal'],\n",
    "             ['flower', 'white'],\n",
    "             ['flower', 'plant'],\n",
    "             ['cactus', 'green'],\n",
    "             ['cactus', 'plant'],\n",
    "             ['dog', 'black'],\n",
    "             ['dog', 'animal'],\n",
    "             ['elephant', 'gray'],\n",
    "             ['elephant', 'animal'],\n",
    "             ['rose', 'red'],\n",
    "             ['rose', 'plant'],\n",
    "             ['car', 'red'],\n",
    "             ['dish', 'red'],\n",
    "             ['jasmine', 'plant'],\n",
    "             ['jasmine', 'white']]\n",
    "\n",
    "\n",
    "word_list = set(np.array(word_pair).flatten())\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "skip_grams = [[word_dict[word[0]], word_dict[word[1]]] for word in word_pair]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "label = th.LongTensor(skip_grams)[:, 0].contiguous()\n",
    "context = th.LongTensor(skip_grams)[:, 1].contiguous()\n",
    "skip_grams_dataset = data_utils.TensorDataset(label, context)\n",
    "train_loader = th.utils.data.DataLoader(skip_grams_dataset, batch_size=8, shuffle=True)\n",
    "test_loader = th.utils.data.DataLoader(skip_grams_dataset, batch_size=1, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class Word2Vec(nn.Module) :\n",
    "    def __init__(self):\n",
    "        super(Word2Vec, self).__init__()\n",
    "        self.embedding = nn.Embedding(len(word_list), 2)\n",
    "        self.linear = nn.Linear(2, len(word_list), bias=True)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.embedding(x)\n",
    "        x = self.linear(x)\n",
    "        return F.log_softmax(x)\n",
    "    \n",
    "model = Word2Vec()\n",
    "loss_fn = nn.NLLLoss() \n",
    "optimizer = th.optim.Adam(model.parameters(), lr=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model.train()\n",
    "for epoch in range(100):\n",
    "    for data, target in train_loader:\n",
    "        data, target = Variable(data), Variable(target) #(입력 생성)\n",
    "        output = model(data) # model 생성\n",
    "        loss = F.nll_loss(output, target) #loss 생성\n",
    "        optimizer.zero_grad() # zeroGrad\n",
    "        loss.backward() # calc backward gradients\n",
    "        optimizer.step() # update parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input : true : pred\n",
      "cat : white : animal\n",
      "cat : animal : animal\n",
      "flower : white : plant\n",
      "flower : plant : plant\n",
      "cactus : green : plant\n",
      "cactus : plant : plant\n",
      "dog : black : animal\n",
      "dog : animal : animal\n",
      "elephant : gray : animal\n",
      "elephant : animal : animal\n",
      "rose : red : red\n",
      "rose : plant : red\n",
      "car : red : red\n",
      "dish : red : red\n",
      "jasmine : plant : plant\n",
      "jasmine : white : plant\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "\n",
    "invDic = { i : w for w, i in word_dict.items()}\n",
    "print('Input : true : pred')\n",
    "\n",
    "for x, y in test_loader :\n",
    "    x, y = Variable(x.squeeze()), y.squeeze()\n",
    "    y_pred = model(x).max(1)[1].data[0][0]\n",
    "    print('{:s} : {:s} : {:s}'.format(invDic[x.data[0]], invDic[y[0]], invDic[y_pred]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD7CAYAAACRxdTpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH+BJREFUeJzt3Xt0VeW57/Hvk6xwv0siYpBbuAQxQBKUixwQyOYUKfZY\nRSm14tBC0V1RR2lxoxZvZe/qUWyRKgd3PdXKGN5QpO59Nsi2oggSInKtxWspFwOiIAEMSZ7zR1ZW\nAwZYiTNrraz8PmNkwHzXnO98FiP8MvPOd73T3B0REUkuKfEuQEREgqdwFxFJQgp3EZEkpHAXEUlC\nCncRkSSkcBcRSUIKdxGRJKRwFxFJQgp3EZEkFIrXiTt27OjdunWL1+lFRBqkDRs27Hf39DPtF7dw\n79atG4WFhfE6vYhIg2Rmn0azn4ZlRESSkMJdkkKPHj0oKysLrL8RI0bwxhtvBNafSKzFbVhGJEgf\nffRRoP2tXr060P5EYk1X7pIUMjIyWL9+PUOHDiU/P59+/fqxbt069u/fT1ZWFrm5uYwaNQqA3r17\nM3ToUC666CK6devG1KlTGTRoEF27duXZZ5+N7LNx40bmz59Ply5dyM7OJjs7O9IHwPDhw8nJySEr\nK4uFCxfG4V2LnJrCXZJCWVkZ3bt3p2XLlpSWllJSUsK8efPYsGED7k5hYSGvv/46ABUVFVx33XWs\nW7eOoUOHsmnTJt59912eeuopfvnLX0b2OX78OABNmjRh+/btbN++na1bt7J7927uueceWrduzaZN\nm9iyZUvkOJFEoXCXpFFQUEBOTg6bNm3i2muv5fDhw4wbN46bbrqJgQMHMnPmzMi+gwYNAqBdu3YM\nHDgQgLPPPptjx459o9/u3btH/t62bVt27drF2rVr2bZtG0OGDGHUqFGEQhrhlMSicJeksWvXLmbP\nng3A8uXLI+233XYbmzZt4plnnmH37t3fOC4lpfb/DS644ALy8vJYu3Yta9euZc+ePXUvXKQeKNwl\nacyaNYvBgwfTv39/WrRogZmxfPlyevbsSXZ2NhkZGXTu3JmUlBTS0tIACIVCkavuUChEamoqQGSf\n6m3V2++//34++ugjevfuzfnnn8+NN94Y+zcschoWr2eo5ufnuz7EJEE566yz+Pzzz2N2vlVPvsR7\nK56louwQKaE2DCiYxOip34vZ+aXxMrMN7p5/pv0CGyg0s4VAGtAS+Ku7zw2qb5HTGTFiBOnpZ/w0\ndmBWPfkS7/7Hk0DlvPqKskPhbRTwkjACG5Zx9xvd/cfu/gOgu5n1CapvkdNZvXo1f/nLXwCYM2cO\nmZmZjBkzhkWLFtXL+d5b8SxVwf4PZeF2kcQQ+Ji7mbUH0oHPgu5b5ExeeOEFnnvuOcrLywP9xGp1\nFWWHatUuEg+BhbuZZZnZH4EiYJG7f1nDPtPMrNDMCvft2xfUqUUAuPbaa/n444+ZOnUqH3zwQaR9\n9OjR9OrVi759+5Kbm8uxY8fo0aMHO3bsAODuu++mZcuWkf1HjBjB8uXLeeGFF+jRowe5ubkMGDCA\n0tJS1q1bx33L/5t7XlnJM2s3nnD+lFCb2LxRkSgEfkPVzELAEuCn7r73VPvphqrUh6ysLJYsWcKs\nWbOYNGkSX3zxBcuXL+ftt98G4Ec/+hEA55xzDnv27OEPf/gD559/PpmZmUydOpXJkyeTmZnJ3//+\ndzIyMtiyZQsZGRn84Ac/4Nxzz+Wyyy5j9CWXcPfEcTQ54Y5ViEHfmaoxd6l3Mb+hWsXdy8wsFWgS\ndN8itfXWW28xceLEyPbkyZP5xS9+wYIFC+jXrx/FxcWkpaUxd+5cfvazn3HkyJHIB5y+/PLLyLFf\nffUVbdu2BeC8rl256Ls3aLaMJLRAwt3McoHbgMNAG+AFd/9bEH2LfBsjRoxg2bJl3H777QAsWbKE\nvLw82rRpQ+vWrZk+fTrXXHMNQ4cOZe/evSxcuJAnnngCgDZt2vDqq6/SoUOHSH9vvvkmqampjJ76\nPYW5JLRAwt3di4AfBtGXyLdR9SGj1NRUQqEQt99+O6+99hq9evUiNTWVVq1asWbNGgB+/OMfM3v2\nbJ577jkAcnJyWLduXWQ5gl/96lfk5OTQpk0bysrKWLNmDWlpaXX6RKtIrOlDTCIiDUi0Y+66BBGJ\nwutP3MNbF/Vna99s3rqoP68/cU+8SxI5LYW7yBm8/sQ9tJu/hA4Hy0kBOhwsp938JQp4SWgKd5Ez\nSFv0LE2Pn9jW9Hhlu0iiUriLnEG7g+W1ahdJBAp3kTP4sm1qrdpFEoHCXeQMjk+bxNdpJ7Z9nVbZ\nLpKoFO4iZzDq+rv48pbJHGibSgVwoG0qX94ymVHX3xXv0kROSfPcRUQaEM1zFxFpxBTuIiJJSOEu\nIpKEFO4iIklI4S4ikoQU7iIiSUjhLiKShBTuIiIB6Nu3L5s2bYp3GREKdxGRAJSVlfH111/Hu4wI\nhbuISDXf//73yc7OJi8vjxkzZjB06FDy8/Pp168f69atA+CBBx4gKyuLvLw8Jk+ezA033MDOnTv5\n4Q9/yPPPP8+YMWNYtGhRpM+MjAwAbrvtNvr06UN+fv4Jr9eHQJ6hKiKSDObPn09xcTHbt28HYP/+\n/Vx99dUUFxdTUlLCvHnzWLBgAQ8++CB79uw54Xm6r7/+Ok8//TSDBw9mwYIFlJWVRV6r+vvSpUt5\n7LHHGDduXL2/l8Cu3M3sd2b2qJktMTM9LFtEGpyVK1cyfvz4yHZBQQE5OTls2rSJa6+9lsOHD7Nq\n1Sp69uxZqwelV63htXr1ah588EHy8vJ4//33A6+/usDC3d1nuPtNwA+A6UH1KyISKyNHjuTFF1+M\nbO/atYvZs2cDsHz58sg+77//PqWlpSccm5KSEhlzb9euHR988AEAb7/9NkeOHAEgMzOTFStWMG3a\nNK655pp6fS/1MSzTFDhQD/2KiNSrWbNmsWrVKrKysmjVqhUzZ85k8ODBtG7dmnbt2mFmdO3alZkz\nZ5KVlUX79u3p168fS5YsYdSoUVx55ZXccMMNzJ07l3HjxrFy5UpCoVBkzH3s2LHs3r2bQ4cOMWfO\nnHp9L4Ev+WtmDwJL3f2t0+2nJX9FRGov2iV/A71yN7NbgXdPFexmNg2YBnDeeecFeWoRkYS16smX\neG/Fs1SUHSIl1IYBBZMYPfV79XrOIG+o3giUuPsfT7WPuy9y93x3z09PTw/q1CIiCWvVky/x7n88\nSUXZIQAqyg7x7n88yaonX6rX8wYS7mY2DJgN5JrZY+EvpbeINHrvrXgWKDuptSzcXn8CGZZx9zWA\nxllERE5SdcUebXtQ9AlVEZF6lBJqU6v2wM5br72LiDRyAwom8c1BklC4vf5o+QERkXpUNSsm1rNl\nAp/nHi3NcxcRqb1o57lrWEZEJAkp3CVm1q1bR9euXcnMzGT06NH06NGDfv36cd5553HXXXcBcMUV\nV3D++eeTn5/Pq6++ypEjR8jJyWHgwIH07Nkzsr6HiJyextwlZo4fP85nn33GgQMH6NevHwsWLGDC\nhAmUlZVxzjnncN111/HGG2/w5z//mezsbAAmT57MqFGj+M1vfsPf/vY3hg0bxt///vc4vxORxKdw\nl5jKzMykRYsWHDp0iAkTJgAQCoXIyspi9erVrFy5kmnTplFeXs6yZcvYvHkz5eXlDBkyBIDmzZvH\ns3yRBkPDMhJTqampQOWSqFVDLGVlZXz44YeMHj2anJwcVq9ezaBBg/jnf/5nsrKy+N73vsfatWtZ\nu3YtO3bsiGf5Ig2GrtwlZtLS0iIPOHj++ee58sor+fnPf05JSQk33ngjmZmZDBw4kLKyMr744gsW\nL17M4MGDGTp0KEuXLiUlJYUpU6bU+1KpIslAUyElYa1f9jhdih4gw/dRbOnszJ3F4Il6Dow0bnFZ\n8lckKOuXPU7/DXfQ3ErBoBP7aLvhDtaDAl4kChpzl4TUpeiBymCvprmV0qXogThVJNKwKNwlIWX4\nvlO0749xJSINk8JdElLxKR4HUGwdY1yJSMOkcJeEtDN3Fke9yQltR70JO3NnxakikYZF4S4JafDE\n6WzJu4+9pFPhxl7S2ZJ3n26mikRJUyFFRBoQrQopItKIKdxFRJKQwl1EJAkFFu5mlmpm95vZfwbV\np4iI1E2QV+4TgGVoSQMRkbgLLIjd/WUAMwuqSxERqaOYjrmb2TQzKzSzwn37av54uYiIfHsxDXd3\nX+Tu+e6en55e88fLRRqLGTNmcPPNN8e7DElSGh8XiZPjx4/HuwRJYvUR7vqOlUbv+9//Ptu2baNF\nixZceOGFbNy4kePHj3PkyBF+//vfs2HDBpYuXYqZ0bp1a37961/Hu2RJMoGHu7t/J+g+RRqS+fPn\nU1xczPbt2wHYv38/V199NcXFxZSUlDBv3jxeeuklioqKSEtLU7BLvdCHmEQCtnLlSsaPHx/ZLigo\nICcnh02bNnHttddy+PDhOFYnjYXCXSRgI0eO5MUXX4xs79q1i9mzZwOwfPnySHsoFNK4u9QbhbtI\nwGbNmkXHjh3Jyspi4MCBzJw5k8GDB9O/f39atGgR+SzI+PHjef7557n88svjXLEkIy35KxIj65c9\nTpeiB8jwfRRbOjtzZ2l9eqm1aJf81VRIkRhYv+xx+m+4o/Kh3wad2EfbDXewHhTwUi80LCMSA12K\nHqgM9mqaWyldih6IU0WS7BTuIjGQ4TUvt5Hh+2NciTQWCneRGCi2mpfbKLaOMa5EGguFu0gM7Myd\nxVFvckLbUW/CztxZcapIkp3CXSQGBk+czpa8+9hLOhVu7CWdLXn36Waq1BtNhRQRaUCinQqpK3cR\nkSSkcBcRSUIKdxGRJKRwFxFJQgp3EZEkpHAXEUlCCncRkSSkcBcRSUIKdxGRJBTYeu5mNgW4CigH\n3nZ3PfVXRCROArlyN7PWwDXAZe7+v4ALzKxXEH2LiEjtBTUsMwxY4f9YqOZl4JKA+hYRkVoKKtzP\nAg5U2z4QbjuBmU0zs0IzK9y3r+aHF4iIyLcXVLh/DrSvtt0h3HYCd1/k7vnunp+eXvPDC0RE5NsL\nKtzXAWPNzMLbE4E3AupbRERqKZDZMu7+pZk9BSwxszJgo7v/JYi+RUSk9gKbCunuS4AlQfUnIiJ1\npw8xiYgkIYW7iEgSUriLiCQhhbuISBJSuIuIJCGFu4hIElK4i4gkIYW7iEgSUriLiCQhhbuISBJS\nuIuIJCGFu4hIElK4i4gkIYW7iEgSUriLiCQhhbuISBJSuIuIJCGFu4hIElK4i4gkIYW7iEgSiuoB\n2WZ2BzC2puPd/eLwPgXAvwMXuvue4EoUEZHaiircgYPAXe7+RlWDmXUG7gz/vR0wAHgNSA26SBER\nqZ1oh2X2AB+bWTcAM2sBdKcyzHH3L939wfooUEREai/qMXd33wkcDgd7pru/VduTmdk0Mys0s8J9\n+/bV9nAREYlSbW+olgJpwOG6nMzdF7l7vrvnp6en16ULERGJQtThbmZtgJ7ufhD42sz61l9ZIiLy\nbUQb7i2A0e7+LoC7fx5uH3DSfuXhLxERiaNoZ8tcAvQ0s1tOau9LeMYMgLtfH1RhIiJSd1GFu7tf\nV9+FiIhIcPQJVRGRJKRwF5GENm7cOBYsWBDvMhochbuIJLTy8nLKysriXUaDE+0NVRGRmBk4cCBH\njx6ladOm7N69mwkTJjB69Gh27txJamoqLVq0YM2aNTRr1ozBgwdz7NgxQqEQxcXFXH755fz2t7+N\n91uIO4W7iCSUJ554glAoxPvvvw9Aeno6JSUlHD16lB07dgDwox/9iGnTpvFP//RPHD9+nM2bNwPQ\nq1cvXeWHaVhGRBLKpk2byMnJiWz37t2bt956i4kTJ0baJk+ezMaNG1m/fj0DBw6MtOfl5cW01kSm\ncBeRhDJs2DDee++9yPb27dsZMWIEy5Yti7QtWbKEvLw8hg0bRlFRUaR9/fr1Ma01kWlYRkQSylVX\nXcXChQvJzs6mSZMmtG/fntatW9OyZUt69epFamoqrVq1Ys2aNTRp0oSnn36a7OxsAL766is6dOgQ\n53eQGMzd43Li/Px8LywsjMu5RST5VFRUkJGRwX/913+Rm5sb73LqjZltcPf8M+2nYRkRabDWL3uT\nzM7n0qlTJ9I7plMwbHRSB3ttaFhGRBqk9cve5P9tWMUN034caQt5CuuXvcngiRfHsbLEoCt3EWmQ\n/lz0JmVWcUJbmVXw56I341RRYlG4i0iDdNiP1aq9sVG4i0iD1Mqa1aq9sVG4i0iDNDL3YkJ+YoSF\nPIWRuRpvB91QFZEGquqm6Z+L3uSwH6OVNWNk3sW6mRqmcBeRBmvwRIX5qWhYRkQkCSncRUSSkMJd\nRCQJRTXmbmZ3AGNrOt7dLzaza4Dh4bYmwHR3Px5QjSIiUkvR3lA9CNzl7m9UNZhZZ+BOAHd/Cngq\n3D4HKABeDbZUERGJVrTDMnuAj82sG4CZtQC6A6/VsG868HEQxYmISN1EPebu7juBw+Fgz3T3t07e\nx8wuqtzVt9fUh5lNM7NCMyvct29fnYsWEZHTq+0N1VIgDTh88gtm1hf4CfDzUx3s7ovcPd/d89PT\n02t5ahERiVbU4W5mbYCe7n4Q+Doc5lWvZQFzgBm6kSoiEn/RhnsLYLS7vwvg7p+H2weE/1we/nO+\nmT1mZhNP7kBERGIn2tkylwA9zeyWk9r7Ane6e98ajhERkTiJKtzd/br6LkRERIKjT6iKiCQhhbuI\nSBJSuIuIJCGFu4hIElK4i4gkIYW7iEgSUriLiCQhhbuISBJSuIuIJCGFu4hIElK4i4gkIYW7iEgS\nUriLiCQhhbuISBJSuIuIJCGFuyS93r17s3HjxlofF9RzfseOHcvixYsD6UskWgp3SXoVFRUcP177\nR/uWl5cHcv6ysjJKS0sD6UskWtE+Zk+kwRg+fDhfffUVR44c4bbbbou0HzlyhCFDhpCSksJXX33F\nI488woQJEzjrrLPIysqiXbt2fPLJJyxZsoTc3FwAcnJyaNasGbt27eIPf/gDY8aMYeHChTz55JNU\nVFRQUVHB2rVreeedd7j88svp0KEDaWlpVFRUsHXrVubMmUNhYSEffvghpaWl3HLLyU+qFKkn7h6X\nr7y8PBcJ2t133+3jxo1zd/ejR496x44dvWfPnv7OO+/41Vdf7T/96U/d3f3TTz/1c889193dmzRp\n4h999JG7u7/44os+YMAAd3dv166db9myxd3dFy5c6EOGDHF39zfffNMHDRrkgwYN8tatW/szzzzj\nq1ev9rZt20bqyM7O9pdeesnd3UeOHOmPPvpoDN69NAZAoUeRsVFduZvZHcDYGl4KufvFZjYZKAAq\ngLbAze6+J7CfQCJRWrt2Ldu2bWPIkCEAhEL/+BbfvHkz5eXlkdeaN28OQLNmzejevTsAl1xyCTNm\nzADAzDj//PMBuOCCC/jiiy8AuPTSS3nuuecoKCggPz+fL774gi5dunD22WdHzpWRkcGuXbvq+d2K\nnFq0wzIHgbvc/Y2qBjPrDNwJ4O5LgCXh9kuAm4Hbgy1V5MwuuOACmjZtytKlSyNtWVlZkT+zs7OZ\nN2/eCcccPXqUoqIicnNzeeqpp+jVq1eNfVdeNFUqKCigrKyMbdu2nbGm1NRUjblLzEUb7nuAj82s\nm7t/YmYtgO7AazXsOxzYGlSBdfGb3/yGrVu30rJlS3Jzc9m6dSulpaWUlJQwZ84cunTpwhVXXMFZ\nZ51Fs2bNeOSRR+JZrgTo/vvvJy8vj969e5OWlsbIkSNJSUkhLS2NxYsXM3ToUJYuXUpKSgpTpkxh\nzpw5NGvWjBkzZlBeXs7+/ftZsWIFUBnKVdLS0khJqZx/MHz4cLKzsykvL6dbt26Y2QmvVx1b9VvD\n+PHjuffee9m2bRuLFi2K4b+GNGZW/WrklDuZXeHuz5tZR+AIkOnuf61qD+8zDxgD7AB+6DV0bGbT\ngGkA5513Xt6nn34a4Fup9Prrr/Paa69x7733ApU30R566CE+++wz9u/fz4UXXsitt95K7969WbFi\nBV27dg28BmlYOnTowIEDB+JdhkhUzGyDu+efab/aToUsBdKAwye/4O63u/uFwHLgpzUd7O6L3D3f\n3fODmkN8ssLCQkaMGBHZ/pd/+Rf69OnDb3/7W6ZMmUJJSQkAnTt3VrALcOIVelCeeH8F/f97FZ1W\nFdH/v1fxxPsrAj+HyOlEHe5m1gbo6e4Hga/NrO8pdj0KtAmiuLoYNGhQ5NdqgB07djB+/HgAXnvt\nH6NI1W+0SeO2b9++QPt74v0V3L27DfvpAJbCfjpw9+42CniJqWgTrgUw2t1fAnD3z80sHRgAPG9m\ni6mcKZMCHANm1Uex0RgzZgyFhYVMnTqVtm3bcvPNN3PDDTfQvn17OnTogJkB0KRJk3iVKEnu4d2p\nlNL0hLZSmvLw7lSu7xOnoqTRiXbM/fdATyoDvLq+7t6pLifOz8/3wsLCuhxaZ3/66E88UvQIe0v2\n0qllJ2bmzuTSHpfGtAZJfp1WFYHV8EuxV7B3dG7sC5KkEu2Ye1RX7u5+3bcvKb7+9NGfmLtmLsfK\njwGwp2QPc9fMBVDAS6A62peVQzI1tIvESqNZW+aRokciwV7lWPkxHinSNEgJ1q2dy2nC1ye0NeFr\nbu0czFo1ItFoNOG+t2RvrdpF6ur6PgX8svMhOnIAvIKOHOCXnQ9xfZ+CeJcmjUijmTLSqWUn9pR8\nc0WETi3rdMtA5LSu71Ogm6cSV43myn1m7kyapTY7oa1ZajNm5s6MU0UiIvWn0Vy5V9001WwZEWkM\nEjrc582bx7Bhw3j55Zd56KGHvvH6vffey9ixYxk6dGhU/V3a41KFuYg0Cgk9LFNeXo671xjsVa8H\n9bQcEZFkknBX7jNnzqSkpITmzZvz17/+la5duzJ9+nQef/xxbrrpJkKhEKFQiH/7t38D4OGHH2bZ\nsmXs2rWLSZMmcdlll8X5HYiIxF9Chfu6desIhUKRhwl/5zvfiVydHz58mA8//JBXXnmFtLS0yDFX\nXXUVkyZNorS0lPHjxyvcRURIsGGZTz75hP79+0e2q55jCdCqVSt+9atf8Ytf/IJHH3000t65c2dA\na8WIiFSXUOHep08fNm/eHNleu3btCa/n5uby0EMP8d5770X1BBwRkcYqoYZlBg4cyNKlS5k+fTpN\nmzbl3HPPpWfPnrz99tvs37+fW265hdatW3P06FG6d+9OamrqN56WIyIiUa4KWR++7aqQB195heKH\n51O2Zw+hc84h49ZbaPvd7wZYoYhI4gl0VchEc/CVV9hz5134scqFwMp272bPnXcBKOBFREiwMfdo\nFT88PxLsVfzYMYofnh+nikREEkuDDPeyPd9cAOx07SIijU2DDPfQOefUql1EpLFpkOGecestWLMT\nV3i0Zs3IuPWWOFUkIpJYGuQN1aqbppotIyJSs6jC3czuAMbWdLy7X1xtv6nAfe6eGUx5p9b2u99V\nmIuInEK0V+4Hgbvc/Y2qBjPrDNxZbfsioC3wl0ArFBGRWot2zH0P8LGZdQMwsxZAd+C18PbZwFXu\nrqdNi4gkgKhvqLr7TuBwONgz3f0tADNLA+4If52WmU0zs0IzK9y3b19daxYRkTOo7WyZUiANOFyt\nbQjQCXjIzB4D+prZ78ys1ckHu/sid8939/z09PQ6Fy0iIqcX9WwZM2sD9HT3d80sZGZ9Adx9NbC6\n2n4r3X1G8KWKiEi0or1ybwGMdvd3Adz983D7gBr2/TqIwkREpO6ivXK/BOhpZid/Sqgv1WbMALi7\nnkAtIhJnUYW7u19X34WIiEhw4raeu5ntAz79lt10BPYHUE59UG11o9rqLpHrU211U1NtXd39jDNS\n4hbuQTCzwmgWrY8H1VY3qq3uErk+1VY336a2BrlwmIiInJ7CXUQkCTX0cF8U7wJOQ7XVjWqru0Su\nT7XVTZ1ra9Bj7iIiUrOGfuUuIiI1SPiHdZxpLXkzuxPIA/aG2//d3d9JlPqq7TeVGK11H21tZjYZ\nKAAqqFyu+WZ3j8mDaKOo7RpgeLitCTDd3Y8nQm3hfQqAfwcujNW/2amY2RTgKqAceNvdfx3Peqoz\ns1TgHiDP3f9nvOs5mZn9jsrv/w7An9z96TiXFGFmC6lcy6sl8Fd3n1urDtw9ob+AnwL/46S2zsDv\nwn+fC1ycqPWFty8CZgIrE622au2XAPMStLY5wPhEqQ1oB/wMeJLKFVLj8r0XrqU18J/8Y4j1KaBX\nPGs6qb7Lwt//Mf3er0OdBqyOdx2nqe//An1qc0xDGJY57VrywDHgh2a22MzmmVnzRKovzmvdn+nf\nrrrhwNaYVVa72tKBj2NW2Rlqc/cv3f3BGNZzOsOAFR5OAOBlKn9QJwR3f9nd18W7jig0BQ7Eu4ia\nmFl7Kv8PfFab4xpCuOOnWEs+/Nq/uvtP3P0GYDNwe6LUV5u17mNdW5XwD8R3gGzgj4lUW7i+iyp3\n9e2JVluCOIsTQ+lAuE1q5z4gYYazAMwsy8z+CBQBi9z9y9oc3yDCPaymteRP9jKQE5tyvuFbrXUf\nh9oAcPfb3f1CYDmVwxGxdsrawstK/wT4eayLCovmey7ePgfaV9vuEG6TKJnZrcC7ifYD3N0/cPcp\nQC9gipl1qs3xDSLcq60lfxD4umot+RqMBGJ2M7XKqepz99XufmX4N4ufAH9x9xnuHrOwqMW/3VGg\nTazqgtPXZmZZVI61z/AY3UiNtrYEsw4Ya2YW3p4IvHGa/aUaM7sRKHH3mP7WWhvuXgakUjmxIGoN\nIdxPu5a8mf3YzP6PmS0CxgGxHgtN5LXuz/Rvt9jMFpnZYipnh/zvRKmNyt8kAOab2WNmNjGBaqtS\nHv6Km/Cv6k8BS8zsaWCTuyfiQ+pj/gP6TMxsGDAbyA1/jz1mZgnxiDgzyzWzp8M1PQO84O5/q1Uf\n/7gPk5jM7PdATyqnK1XX191r9WtKfUjk+lRb3SRybSLRSvhwFxGR2msIwzIiIlJLCncRkSSkcBcR\nSUIKdxGRJKRwFxFJQgp3EZEkpHAXEUlC/x8cHcPWOSdPjwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa0a9cebda0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in label :\n",
    "    x = Variable(th.LongTensor([i]))\n",
    "    fx, fy = model.embedding(x).squeeze().data\n",
    "    plt.scatter(fx, fy)\n",
    "    plt.annotate(invDic[i], xy=(fx, fy), xytext=(5, 2),\n",
    "                 textcoords='offset points', ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}