word2vec_skipgram_medium_v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    # obtains tokens with a least 1 alphabet\n",
    "    pattern = re.compile(r'[A-Za-z]+[\\w^\\']*|[\\w^\\']*[A-Za-z]+[\\w^\\']*')\n",
    "    return pattern.findall(text.lower())\n",
    "\n",
    "def mapping(tokens):\n",
    "    word_to_id = dict()\n",
    "    id_to_word = dict()\n",
    "\n",
    "    for i, token in enumerate(set(tokens)):\n",
    "        word_to_id[token] = i\n",
    "        id_to_word[i] = token\n",
    "\n",
    "    return word_to_id, id_to_word\n",
    "\n",
    "def generate_training_data(tokens, word_to_id, window_size):\n",
    "    N = len(tokens)\n",
    "    X, Y = [], []\n",
    "\n",
    "    for i in range(N):\n",
    "        nbr_inds = list(range(max(0, i - window_size), i)) + \\\n",
    "                   list(range(i + 1, min(N, i + window_size + 1)))\n",
    "        for j in nbr_inds:\n",
    "            X.append(word_to_id[tokens[i]])\n",
    "            Y.append(word_to_id[tokens[j]])\n",
    "            \n",
    "    X = np.array(X)\n",
    "    X = np.expand_dims(X, axis=0)\n",
    "    Y = np.array(Y)\n",
    "    Y = np.expand_dims(Y, axis=0)\n",
    "            \n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = \"After the deduction of the costs of investing, \" \\\n",
    "      \"beating the stock market is a loser's game.\"\n",
    "tokens = tokenize(doc)\n",
    "word_to_id, id_to_word = mapping(tokens)\n",
    "X, Y = generate_training_data(tokens, word_to_id, 3)\n",
    "vocab_size = len(id_to_word)\n",
    "m = Y.shape[1]\n",
    "# turn Y into one hot encoding\n",
    "Y_one_hot = np.zeros((vocab_size, m))\n",
    "Y_one_hot[Y.flatten(), np.arange(m)] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_wrd_emb(vocab_size, emb_size):\n",
    "    \"\"\"\n",
    "    vocab_size: int. vocabulary size of your corpus or training data\n",
    "    emb_size: int. word embedding size. How many dimensions to represent each vocabulary\n",
    "    \"\"\"\n",
    "    WRD_EMB = np.random.randn(vocab_size, emb_size) * 0.01\n",
    "    \n",
    "    assert(WRD_EMB.shape == (vocab_size, emb_size))\n",
    "    return WRD_EMB\n",
    "\n",
    "def initialize_dense(input_size, output_size):\n",
    "    \"\"\"\n",
    "    input_size: int. size of the input to the dense layer\n",
    "    output_szie: int. size of the output out of the dense layer\n",
    "    \"\"\"\n",
    "    W = np.random.randn(output_size, input_size) * 0.01\n",
    "    \n",
    "    assert(W.shape == (output_size, input_size))\n",
    "    return W\n",
    "\n",
    "def initialize_parameters(vocab_size, emb_size):\n",
    "    WRD_EMB = initialize_wrd_emb(vocab_size, emb_size)\n",
    "    W = initialize_dense(emb_size, vocab_size)\n",
    "    \n",
    "    parameters = {}\n",
    "    parameters['WRD_EMB'] = WRD_EMB\n",
    "    parameters['W'] = W\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forward Propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ind_to_word_vecs(inds, parameters):\n",
    "    \"\"\"\n",
    "    inds: numpy array. shape: (1, m)\n",
    "    parameters: dict. weights to be trained\n",
    "    \"\"\"\n",
    "    m = inds.shape[1]\n",
    "    WRD_EMB = parameters['WRD_EMB']\n",
    "    word_vec = WRD_EMB[inds.flatten(), :].T\n",
    "    \n",
    "    assert(word_vec.shape == (WRD_EMB.shape[1], m))\n",
    "    \n",
    "    return word_vec\n",
    "\n",
    "def linear_dense(word_vec, parameters):\n",
    "    \"\"\"\n",
    "    word_vec: numpy array. shape: (emb_size, m)\n",
    "    parameters: dict. weights to be trained\n",
    "    \"\"\"\n",
    "    m = word_vec.shape[1]\n",
    "    W = parameters['W']\n",
    "    Z = np.dot(W, word_vec)\n",
    "    \n",
    "    assert(Z.shape == (W.shape[0], m))\n",
    "    \n",
    "    return W, Z\n",
    "\n",
    "def softmax(Z):\n",
    "    \"\"\"\n",
    "    Z: output out of the dense layer. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    softmax_out = np.divide(np.exp(Z), np.sum(np.exp(Z), axis=0, keepdims=True) + 0.001)\n",
    "    \n",
    "    assert(softmax_out.shape == Z.shape)\n",
    "\n",
    "    return softmax_out\n",
    "\n",
    "def forward_propagation(inds, parameters):\n",
    "    word_vec = ind_to_word_vecs(inds, parameters)\n",
    "    W, Z = linear_dense(word_vec, parameters)\n",
    "    softmax_out = softmax(Z)\n",
    "    \n",
    "    caches = {}\n",
    "    caches['inds'] = inds\n",
    "    caches['word_vec'] = word_vec\n",
    "    caches['W'] = W\n",
    "    caches['Z'] = Z\n",
    "    \n",
    "    return softmax_out, caches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cost Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(softmax_out, Y):\n",
    "    \"\"\"\n",
    "    softmax_out: output out of softmax. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    m = softmax_out.shape[1]\n",
    "    cost = -(1 / m) * np.sum(np.sum(Y * np.log(softmax_out + 0.001), axis=0, keepdims=True), axis=1)\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Backward Propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_backward(Y, softmax_out):\n",
    "    \"\"\"\n",
    "    Y: labels of training data. shape: (vocab_size, m)\n",
    "    softmax_out: output out of softmax. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    dL_dZ = softmax_out - Y\n",
    "    \n",
    "    assert(dL_dZ.shape == softmax_out.shape)\n",
    "    return dL_dZ\n",
    "\n",
    "def dense_backward(dL_dZ, caches):\n",
    "    \"\"\"\n",
    "    dL_dZ: shape: (vocab_size, m)\n",
    "    caches: dict. results from each steps of forward propagation\n",
    "    \"\"\"\n",
    "    W = caches['W']\n",
    "    word_vec = caches['word_vec']\n",
    "    m = word_vec.shape[1]\n",
    "    \n",
    "    dL_dW = (1 / m) * np.dot(dL_dZ, word_vec.T)\n",
    "    dL_dword_vec = np.dot(W.T, dL_dZ)\n",
    "\n",
    "    assert(W.shape == dL_dW.shape)\n",
    "    assert(word_vec.shape == dL_dword_vec.shape)\n",
    "    \n",
    "    return dL_dW, dL_dword_vec\n",
    "\n",
    "def backward_propagation(Y, softmax_out, caches):\n",
    "    dL_dZ = softmax_backward(Y, softmax_out)\n",
    "    dL_dW, dL_dword_vec = dense_backward(dL_dZ, caches)\n",
    "    \n",
    "    gradients = dict()\n",
    "    gradients['dL_dZ'] = dL_dZ\n",
    "    gradients['dL_dW'] = dL_dW\n",
    "    gradients['dL_dword_vec'] = dL_dword_vec\n",
    "    \n",
    "    return gradients\n",
    "\n",
    "def update_parameters(parameters, caches, gradients, learning_rate):\n",
    "    vocab_size, emb_size = parameters['WRD_EMB'].shape\n",
    "    inds = caches['inds']\n",
    "    dL_dword_vec = gradients['dL_dword_vec']\n",
    "    m = inds.shape[-1]\n",
    "    \n",
    "    parameters['WRD_EMB'][inds.flatten(), :] -= dL_dword_vec.T * learning_rate\n",
    "\n",
    "    parameters['W'] -= learning_rate * gradients['dL_dW']\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def skipgram_model_training(X, Y, vocab_size, emb_size, learning_rate, epochs, batch_size=256, parameters=None, print_cost=False, plot_cost=True):\n",
    "    costs = []\n",
    "    m = X.shape[1]\n",
    "    \n",
    "    if parameters is None:\n",
    "        parameters = initialize_parameters(vocab_size, emb_size)\n",
    "    \n",
    "    begin_time = datetime.now()\n",
    "    for epoch in range(epochs):\n",
    "        epoch_cost = 0\n",
    "        batch_inds = list(range(0, m, batch_size))\n",
    "        np.random.shuffle(batch_inds)\n",
    "        for i in batch_inds:\n",
    "            X_batch = X[:, i:i+batch_size]\n",
    "            Y_batch = Y[:, i:i+batch_size]\n",
    "\n",
    "            softmax_out, caches = forward_propagation(X_batch, parameters)\n",
    "            gradients = backward_propagation(Y_batch, softmax_out, caches)\n",
    "            update_parameters(parameters, caches, gradients, learning_rate)\n",
    "            cost = cross_entropy(softmax_out, Y_batch)\n",
    "            epoch_cost += np.squeeze(cost)\n",
    "            \n",
    "        costs.append(epoch_cost)\n",
    "        if print_cost and epoch % (epochs // 500) == 0:\n",
    "            print(\"Cost after epoch {}: {}\".format(epoch, epoch_cost))\n",
    "        if epoch % (epochs // 100) == 0:\n",
    "            learning_rate *= 0.98\n",
    "    end_time = datetime.now()\n",
    "    print('training time: {}'.format(end_time - begin_time))\n",
    "            \n",
    "    if plot_cost:\n",
    "        plt.plot(np.arange(epochs), costs)\n",
    "        plt.xlabel('# of epochs')\n",
    "        plt.ylabel('cost')\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after epoch 0: 2.552025554661277\n",
      "Cost after epoch 10: 2.5517266356474195\n",
      "Cost after epoch 20: 2.551408290101263\n",
      "Cost after epoch 30: 2.551050891535881\n",
      "Cost after epoch 40: 2.5506343277669057\n",
      "Cost after epoch 50: 2.550136846741796\n",
      "Cost after epoch 60: 2.5495463562089244\n",
      "Cost after epoch 70: 2.5488309596013536\n",
      "Cost after epoch 80: 2.5479611703175884\n",
      "Cost after epoch 90: 2.5469047766303037\n",
      "Cost after epoch 100: 2.5456251158573084\n",
      "Cost after epoch 110: 2.544110738810103\n",
      "Cost after epoch 120: 2.542299901371372\n",
      "Cost after epoch 130: 2.5401394304637708\n",
      "Cost after epoch 140: 2.537572873124299\n",
      "Cost after epoch 150: 2.534537603979789\n",
      "Cost after epoch 160: 2.5310343657116303\n",
      "Cost after epoch 170: 2.526952358476625\n",
      "Cost after epoch 180: 2.5222126446410464\n",
      "Cost after epoch 190: 2.516742078116592\n",
      "Cost after epoch 200: 2.510470308985515\n",
      "Cost after epoch 210: 2.5034707719066724\n",
      "Cost after epoch 220: 2.4956090463740312\n",
      "Cost after epoch 230: 2.4868462626783083\n",
      "Cost after epoch 240: 2.4771851390272683\n",
      "Cost after epoch 250: 2.466664835597383\n",
      "Cost after epoch 260: 2.455577425782694\n",
      "Cost after epoch 270: 2.4438893549517466\n",
      "Cost after epoch 280: 2.4317466518214914\n",
      "Cost after epoch 290: 2.4193568638906786\n",
      "Cost after epoch 300: 2.4069559735834263\n",
      "Cost after epoch 310: 2.395006895401216\n",
      "Cost after epoch 320: 2.383533417208105\n",
      "Cost after epoch 330: 2.3727044654500173\n",
      "Cost after epoch 340: 2.362665082159256\n",
      "Cost after epoch 350: 2.3534981157634633\n",
      "Cost after epoch 360: 2.3453595520782047\n",
      "Cost after epoch 370: 2.3380409244279363\n",
      "Cost after epoch 380: 2.331422157019992\n",
      "Cost after epoch 390: 2.3253604650909763\n",
      "Cost after epoch 400: 2.3196886493459163\n",
      "Cost after epoch 410: 2.3143283812867903\n",
      "Cost after epoch 420: 2.309023824386466\n",
      "Cost after epoch 430: 2.303618237954263\n",
      "Cost after epoch 440: 2.2979876440144227\n",
      "Cost after epoch 450: 2.2920338021993816\n",
      "Cost after epoch 460: 2.285804029421989\n",
      "Cost after epoch 470: 2.2791670921151397\n",
      "Cost after epoch 480: 2.27209406825276\n",
      "Cost after epoch 490: 2.2645905061009537\n",
      "Cost after epoch 500: 2.256681496800939\n",
      "Cost after epoch 510: 2.2485605799038813\n",
      "Cost after epoch 520: 2.2401578138770577\n",
      "Cost after epoch 530: 2.231517138734631\n",
      "Cost after epoch 540: 2.2227042760252935\n",
      "Cost after epoch 550: 2.2137873792381497\n",
      "Cost after epoch 560: 2.204995627258565\n",
      "Cost after epoch 570: 2.196248243605991\n",
      "Cost after epoch 580: 2.18758417672904\n",
      "Cost after epoch 590: 2.179054930621102\n",
      "Cost after epoch 600: 2.1707060061972436\n",
      "Cost after epoch 610: 2.1627212639138977\n",
      "Cost after epoch 620: 2.1549950961832027\n",
      "Cost after epoch 630: 2.1475363897022506\n",
      "Cost after epoch 640: 2.140365084246692\n",
      "Cost after epoch 650: 2.1334958588248867\n",
      "Cost after epoch 660: 2.1270540729451684\n",
      "Cost after epoch 670: 2.1209300314756194\n",
      "Cost after epoch 680: 2.115111254492575\n",
      "Cost after epoch 690: 2.1095953926427207\n",
      "Cost after epoch 700: 2.1043772915686483\n",
      "Cost after epoch 710: 2.0995360940685956\n",
      "Cost after epoch 720: 2.094975364829135\n",
      "Cost after epoch 730: 2.090675274831648\n",
      "Cost after epoch 740: 2.0866254869107936\n",
      "Cost after epoch 750: 2.0828153144408263\n",
      "Cost after epoch 760: 2.0792967958183417\n",
      "Cost after epoch 770: 2.0759954409614756\n",
      "Cost after epoch 780: 2.07289405737659\n",
      "Cost after epoch 790: 2.069983199771423\n",
      "Cost after epoch 800: 2.0672537915650024\n",
      "Cost after epoch 810: 2.0647419340646374\n",
      "Cost after epoch 820: 2.0623934484254502\n",
      "Cost after epoch 830: 2.0601955149497906\n",
      "Cost after epoch 840: 2.0581409294467097\n",
      "Cost after epoch 850: 2.0562227662438315\n",
      "Cost after epoch 860: 2.054465655982089\n",
      "Cost after epoch 870: 2.0528308130094333\n",
      "Cost after epoch 880: 2.0513086431448415\n",
      "Cost after epoch 890: 2.04989347187254\n",
      "Cost after epoch 900: 2.0485798100042003\n",
      "Cost after epoch 910: 2.0473836400096816\n",
      "Cost after epoch 920: 2.0462775851888555\n",
      "Cost after epoch 930: 2.0452544414464264\n",
      "Cost after epoch 940: 2.044309734250396\n",
      "Cost after epoch 950: 2.043439172668056\n",
      "Cost after epoch 960: 2.042652642907659\n",
      "Cost after epoch 970: 2.0419313838374844\n",
      "Cost after epoch 980: 2.041270197034768\n",
      "Cost after epoch 990: 2.0406657537441055\n",
      "Cost after epoch 1000: 2.040114904529763\n",
      "Cost after epoch 1010: 2.039623412279073\n",
      "Cost after epoch 1020: 2.039178963671529\n",
      "Cost after epoch 1030: 2.0387779989537234\n",
      "Cost after epoch 1040: 2.0384181728471127\n",
      "Cost after epoch 1050: 2.0380972780711923\n",
      "Cost after epoch 1060: 2.0378181882634476\n",
      "Cost after epoch 1070: 2.0375732666545927\n",
      "Cost after epoch 1080: 2.0373601415451\n",
      "Cost after epoch 1090: 2.0371771551392546\n",
      "Cost after epoch 1100: 2.0370227329684205\n",
      "Cost after epoch 1110: 2.0368975827774456\n",
      "Cost after epoch 1120: 2.03679737116234\n",
      "Cost after epoch 1130: 2.036720498536258\n",
      "Cost after epoch 1140: 2.0366657116963447\n",
      "Cost after epoch 1150: 2.036631800248444\n",
      "Cost after epoch 1160: 2.036617818973076\n",
      "Cost after epoch 1170: 2.036621748732628\n",
      "Cost after epoch 1180: 2.0366424750353285\n",
      "Cost after epoch 1190: 2.0366789725095966\n",
      "Cost after epoch 1200: 2.0367302389405224\n",
      "Cost after epoch 1210: 2.036794138506875\n",
      "Cost after epoch 1220: 2.036870270670969\n",
      "Cost after epoch 1230: 2.036957844060551\n",
      "Cost after epoch 1240: 2.0370559833986293\n",
      "Cost after epoch 1250: 2.0371638303220023\n",
      "Cost after epoch 1260: 2.0372784941382767\n",
      "Cost after epoch 1270: 2.0374006812692564\n",
      "Cost after epoch 1280: 2.0375298347499378\n",
      "Cost after epoch 1290: 2.0376652031404006\n",
      "Cost after epoch 1300: 2.037806050724053\n",
      "Cost after epoch 1310: 2.0379491175145037\n",
      "Cost after epoch 1320: 2.038095815019412\n",
      "Cost after epoch 1330: 2.0382457632543862\n",
      "Cost after epoch 1340: 2.0383983252684823\n",
      "Cost after epoch 1350: 2.03855288005625\n",
      "Cost after epoch 1360: 2.0387061195334693\n",
      "Cost after epoch 1370: 2.0388598360255092\n",
      "Cost after epoch 1380: 2.0390137844164156\n",
      "Cost after epoch 1390: 2.0391674391037338\n",
      "Cost after epoch 1400: 2.0393202918549234\n",
      "Cost after epoch 1410: 2.039469245119858\n",
      "Cost after epoch 1420: 2.0396162164717033\n",
      "Cost after epoch 1430: 2.0397610676604048\n",
      "Cost after epoch 1440: 2.0399033880617954\n",
      "Cost after epoch 1450: 2.0400427873353224\n",
      "Cost after epoch 1460: 2.0401765786912796\n",
      "Cost after epoch 1470: 2.0403066147288507\n",
      "Cost after epoch 1480: 2.040432849778052\n",
      "Cost after epoch 1490: 2.0405549976987976\n",
      "Cost after epoch 1500: 2.04067279622823\n",
      "Cost after epoch 1510: 2.0407841081380695\n",
      "Cost after epoch 1520: 2.040890590746958\n",
      "Cost after epoch 1530: 2.0409922849469626\n",
      "Cost after epoch 1540: 2.041089037973653\n",
      "Cost after epoch 1550: 2.0411807233914114\n",
      "Cost after epoch 1560: 2.041265822129225\n",
      "Cost after epoch 1570: 2.0413457178246768\n",
      "Cost after epoch 1580: 2.041420531258188\n",
      "Cost after epoch 1590: 2.0414902420849304\n",
      "Cost after epoch 1600: 2.0415548557505505\n",
      "Cost after epoch 1610: 2.04161346349045\n",
      "Cost after epoch 1620: 2.041667139987817\n",
      "Cost after epoch 1630: 2.041716070864606\n",
      "Cost after epoch 1640: 2.0417603506909945\n",
      "Cost after epoch 1650: 2.0418000954506024\n",
      "Cost after epoch 1660: 2.041834927068741\n",
      "Cost after epoch 1670: 2.0418656109172955\n",
      "Cost after epoch 1680: 2.0418923714880717\n",
      "Cost after epoch 1690: 2.041915383954297\n",
      "Cost after epoch 1700: 2.0419348373359116\n",
      "Cost after epoch 1710: 2.0419507518648685\n",
      "Cost after epoch 1720: 2.041963610731259\n",
      "Cost after epoch 1730: 2.0419736441955227\n",
      "Cost after epoch 1740: 2.041981063707631\n",
      "Cost after epoch 1750: 2.041986085720065\n",
      "Cost after epoch 1760: 2.0419889735536403\n",
      "Cost after epoch 1770: 2.0419899756051607\n",
      "Cost after epoch 1780: 2.04198929583917\n",
      "Cost after epoch 1790: 2.0419871386503656\n",
      "Cost after epoch 1800: 2.0419837056490677\n",
      "Cost after epoch 1810: 2.0419793574095038\n",
      "Cost after epoch 1820: 2.0419741673595944\n",
      "Cost after epoch 1830: 2.0419682905922034\n",
      "Cost after epoch 1840: 2.0419618924920697\n",
      "Cost after epoch 1850: 2.0419551306765937\n",
      "Cost after epoch 1860: 2.0419483511899172\n",
      "Cost after epoch 1870: 2.0419415160389307\n",
      "Cost after epoch 1880: 2.0419347230969076\n",
      "Cost after epoch 1890: 2.0419280836579565\n",
      "Cost after epoch 1900: 2.0419216996097775\n",
      "Cost after epoch 1910: 2.0419158356272997\n",
      "Cost after epoch 1920: 2.041910400841165\n",
      "Cost after epoch 1930: 2.041905441161319\n",
      "Cost after epoch 1940: 2.0419010148341186\n",
      "Cost after epoch 1950: 2.0418971718121774\n",
      "Cost after epoch 1960: 2.0418940706142132\n",
      "Cost after epoch 1970: 2.0418916142064054\n",
      "Cost after epoch 1980: 2.0418898107669827\n",
      "Cost after epoch 1990: 2.041888677514453\n",
      "Cost after epoch 2000: 2.0418882260821434\n",
      "Cost after epoch 2010: 2.041888514379123\n",
      "Cost after epoch 2020: 2.0418894709045285\n",
      "Cost after epoch 2030: 2.041891083568684\n",
      "Cost after epoch 2040: 2.041893345221149\n",
      "Cost after epoch 2050: 2.041896246294314\n",
      "Cost after epoch 2060: 2.0418997656046702\n",
      "Cost after epoch 2070: 2.0419038746451426\n",
      "Cost after epoch 2080: 2.041908557171509\n",
      "Cost after epoch 2090: 2.0419137977782436\n",
      "Cost after epoch 2100: 2.041919581438425\n",
      "Cost after epoch 2110: 2.0419258327213465\n",
      "Cost after epoch 2120: 2.0419325727941606\n",
      "Cost after epoch 2130: 2.041939793307065\n",
      "Cost after epoch 2140: 2.0419474828703543\n",
      "Cost after epoch 2150: 2.041955632395903\n",
      "Cost after epoch 2160: 2.041964131424989\n",
      "Cost after epoch 2170: 2.0419730509718894\n",
      "Cost after epoch 2180: 2.041982397428503\n",
      "Cost after epoch 2190: 2.041992170255589\n",
      "Cost after epoch 2200: 2.042002372055844\n",
      "Cost after epoch 2210: 2.0420128662101633\n",
      "Cost after epoch 2220: 2.042023770973388\n",
      "Cost after epoch 2230: 2.042035109180884\n",
      "Cost after epoch 2240: 2.0420468923959225\n",
      "Cost after epoch 2250: 2.0420591351416295\n",
      "Cost after epoch 2260: 2.042071672537925\n",
      "Cost after epoch 2270: 2.042084667306831\n",
      "Cost after epoch 2280: 2.0420981563103213\n",
      "Cost after epoch 2290: 2.0421121599747556\n",
      "Cost after epoch 2300: 2.0421267007650004\n",
      "Cost after epoch 2310: 2.0421415752023906\n",
      "Cost after epoch 2320: 2.042156988925308\n",
      "Cost after epoch 2330: 2.042172987961695\n",
      "Cost after epoch 2340: 2.0421895957708083\n",
      "Cost after epoch 2350: 2.0422068365830386\n",
      "Cost after epoch 2360: 2.042224454389462\n",
      "Cost after epoch 2370: 2.042242697639121\n",
      "Cost after epoch 2380: 2.0422616160053506\n",
      "Cost after epoch 2390: 2.0422812296023767\n",
      "Cost after epoch 2400: 2.042301558067739\n",
      "Cost after epoch 2410: 2.0423222802087966\n",
      "Cost after epoch 2420: 2.042343687881186\n",
      "Cost after epoch 2430: 2.042365829586499\n",
      "Cost after epoch 2440: 2.0423887167474892\n",
      "Cost after epoch 2450: 2.042412359326878\n",
      "Cost after epoch 2460: 2.042436363376027\n",
      "Cost after epoch 2470: 2.0424610641356526\n",
      "Cost after epoch 2480: 2.0424865055112935\n",
      "Cost after epoch 2490: 2.0425126867583945\n",
      "Cost after epoch 2500: 2.0425396050584474\n",
      "Cost after epoch 2510: 2.0425667933717238\n",
      "Cost after epoch 2520: 2.04259462828335\n",
      "Cost after epoch 2530: 2.042623147153238\n",
      "Cost after epoch 2540: 2.0426523356303994\n",
      "Cost after epoch 2550: 2.042682177043446\n",
      "Cost after epoch 2560: 2.0427121387288225\n",
      "Cost after epoch 2570: 2.0427426334928103\n",
      "Cost after epoch 2580: 2.042773691388723\n",
      "Cost after epoch 2590: 2.0428052847921956\n",
      "Cost after epoch 2600: 2.042837383818977\n",
      "Cost after epoch 2610: 2.0428694048146414\n",
      "Cost after epoch 2620: 2.042901788102516\n",
      "Cost after epoch 2630: 2.042934556444341\n",
      "Cost after epoch 2640: 2.0429676705972075\n",
      "Cost after epoch 2650: 2.043001089345056\n",
      "Cost after epoch 2660: 2.0430341984551044\n",
      "Cost after epoch 2670: 2.043067453573175\n",
      "Cost after epoch 2680: 2.0431008705766103\n",
      "Cost after epoch 2690: 2.043134401107258\n",
      "Cost after epoch 2700: 2.0431679952673414\n",
      "Cost after epoch 2710: 2.043201032679209\n",
      "Cost after epoch 2720: 2.0432339691930603\n",
      "Cost after epoch 2730: 2.0432668143103463\n",
      "Cost after epoch 2740: 2.043299513491367\n",
      "Cost after epoch 2750: 2.0433320111678905\n",
      "Cost after epoch 2760: 2.043363707412548\n",
      "Cost after epoch 2770: 2.0433950411148527\n",
      "Cost after epoch 2780: 2.0434260158197253\n",
      "Cost after epoch 2790: 2.0434565738319366\n",
      "Cost after epoch 2800: 2.0434866569597263\n",
      "Cost after epoch 2810: 2.0435157127078343\n",
      "Cost after epoch 2820: 2.043544144815887\n",
      "Cost after epoch 2830: 2.0435719510963795\n",
      "Cost after epoch 2840: 2.0435990735601868\n",
      "Cost after epoch 2850: 2.0436254542305523\n",
      "Cost after epoch 2860: 2.04365061384511\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after epoch 2870: 2.04367490281571\n",
      "Cost after epoch 2880: 2.0436983132096738\n",
      "Cost after epoch 2890: 2.0437207892601523\n",
      "Cost after epoch 2900: 2.043742275666628\n",
      "Cost after epoch 2910: 2.043762389736028\n",
      "Cost after epoch 2920: 2.0437814113364867\n",
      "Cost after epoch 2930: 2.043799326546258\n",
      "Cost after epoch 2940: 2.0438160838786295\n",
      "Cost after epoch 2950: 2.043831632691763\n",
      "Cost after epoch 2960: 2.043845706774271\n",
      "Cost after epoch 2970: 2.043858499998313\n",
      "Cost after epoch 2980: 2.043869992016681\n",
      "Cost after epoch 2990: 2.043880137169048\n",
      "Cost after epoch 3000: 2.0438888909327244\n",
      "Cost after epoch 3010: 2.0438961194377483\n",
      "Cost after epoch 3020: 2.043901917488309\n",
      "Cost after epoch 3030: 2.043906257742082\n",
      "Cost after epoch 3040: 2.0439091013957156\n",
      "Cost after epoch 3050: 2.043910410988596\n",
      "Cost after epoch 3060: 2.043910196482963\n",
      "Cost after epoch 3070: 2.0439084433410706\n",
      "Cost after epoch 3080: 2.0439051165916133\n",
      "Cost after epoch 3090: 2.0439001848368465\n",
      "Cost after epoch 3100: 2.043893618143868\n",
      "Cost after epoch 3110: 2.0438855773873135\n",
      "Cost after epoch 3120: 2.0438759314348616\n",
      "Cost after epoch 3130: 2.043864637057781\n",
      "Cost after epoch 3140: 2.043851670390313\n",
      "Cost after epoch 3150: 2.0438370090798643\n",
      "Cost after epoch 3160: 2.043820967766999\n",
      "Cost after epoch 3170: 2.043803294539836\n",
      "Cost after epoch 3180: 2.043783937352286\n",
      "Cost after epoch 3190: 2.043762879649417\n",
      "Cost after epoch 3200: 2.043740106375921\n",
      "Cost after epoch 3210: 2.0437160849205838\n",
      "Cost after epoch 3220: 2.043690441370437\n",
      "Cost after epoch 3230: 2.0436631144222335\n",
      "Cost after epoch 3240: 2.0436340943435174\n",
      "Cost after epoch 3250: 2.0436033728390823\n",
      "Cost after epoch 3260: 2.0435715656305025\n",
      "Cost after epoch 3270: 2.043538178322154\n",
      "Cost after epoch 3280: 2.043503140071399\n",
      "Cost after epoch 3290: 2.0434664472913497\n",
      "Cost after epoch 3300: 2.043428097734076\n",
      "Cost after epoch 3310: 2.0433888481834446\n",
      "Cost after epoch 3320: 2.0433480877094556\n",
      "Cost after epoch 3330: 2.043305735827855\n",
      "Cost after epoch 3340: 2.043261794303599\n",
      "Cost after epoch 3350: 2.0432162661183106\n",
      "Cost after epoch 3360: 2.043170039562536\n",
      "Cost after epoch 3370: 2.043122393184615\n",
      "Cost after epoch 3380: 2.0430732369446645\n",
      "Cost after epoch 3390: 2.043022577110611\n",
      "Cost after epoch 3400: 2.0429704210311197\n",
      "Cost after epoch 3410: 2.0429177772262217\n",
      "Cost after epoch 3420: 2.0428638214071317\n",
      "Cost after epoch 3430: 2.042808454248346\n",
      "Cost after epoch 3440: 2.0427516856668304\n",
      "Cost after epoch 3450: 2.0426935265193\n",
      "Cost after epoch 3460: 2.042635093088262\n",
      "Cost after epoch 3470: 2.0425754672080423\n",
      "Cost after epoch 3480: 2.0425145407074154\n",
      "Cost after epoch 3490: 2.042452326331756\n",
      "Cost after epoch 3500: 2.0423888376268895\n",
      "Cost after epoch 3510: 2.042325285463474\n",
      "Cost after epoch 3520: 2.0422606676540402\n",
      "Cost after epoch 3530: 2.0421948677700335\n",
      "Cost after epoch 3540: 2.042127900625319\n",
      "Cost after epoch 3550: 2.042059781701569\n",
      "Cost after epoch 3560: 2.0419918029985236\n",
      "Cost after epoch 3570: 2.041922888705138\n",
      "Cost after epoch 3580: 2.0418529148464217\n",
      "Cost after epoch 3590: 2.0417818976232387\n",
      "Cost after epoch 3600: 2.0417098537817004\n",
      "Cost after epoch 3610: 2.041638143064409\n",
      "Cost after epoch 3620: 2.04156562666472\n",
      "Cost after epoch 3630: 2.0414921738663767\n",
      "Cost after epoch 3640: 2.0414178016638242\n",
      "Cost after epoch 3650: 2.041342527486395\n",
      "Cost after epoch 3660: 2.041267765798162\n",
      "Cost after epoch 3670: 2.041192325384935\n",
      "Cost after epoch 3680: 2.041116069660112\n",
      "Cost after epoch 3690: 2.0410390159096083\n",
      "Cost after epoch 3700: 2.0409611817571074\n",
      "Cost after epoch 3710: 2.0408840239605945\n",
      "Cost after epoch 3720: 2.040806309222394\n",
      "Cost after epoch 3730: 2.040727895995009\n",
      "Cost after epoch 3740: 2.040648801443386\n",
      "Cost after epoch 3750: 2.0405690429865855\n",
      "Cost after epoch 3760: 2.040490108013603\n",
      "Cost after epoch 3770: 2.0404107310243753\n",
      "Cost after epoch 3780: 2.040330766433603\n",
      "Cost after epoch 3790: 2.040250230957411\n",
      "Cost after epoch 3800: 2.0401691414954684\n",
      "Cost after epoch 3810: 2.0400890052883924\n",
      "Cost after epoch 3820: 2.0400085339354876\n",
      "Cost after epoch 3830: 2.039927578728595\n",
      "Cost after epoch 3840: 2.039846155684394\n",
      "Cost after epoch 3850: 2.0397642809448095\n",
      "Cost after epoch 3860: 2.0396834717799703\n",
      "Cost after epoch 3870: 2.0396024255134764\n",
      "Cost after epoch 3880: 2.0395209912027386\n",
      "Cost after epoch 3890: 2.0394391839829202\n",
      "Cost after epoch 3900: 2.0393570190673365\n",
      "Cost after epoch 3910: 2.039276014926526\n",
      "Cost after epoch 3920: 2.039194862493359\n",
      "Cost after epoch 3930: 2.0391134094379364\n",
      "Cost after epoch 3940: 2.0390316698911577\n",
      "Cost after epoch 3950: 2.038949658024967\n",
      "Cost after epoch 3960: 2.0388688856787653\n",
      "Cost after epoch 3970: 2.0387880445061417\n",
      "Cost after epoch 3980: 2.038706981584332\n",
      "Cost after epoch 3990: 2.0386257099673273\n",
      "Cost after epoch 4000: 2.03854424272175\n",
      "Cost after epoch 4010: 2.038464078201653\n",
      "Cost after epoch 4020: 2.0383839151103076\n",
      "Cost after epoch 4030: 2.03830360066574\n",
      "Cost after epoch 4040: 2.03822314681363\n",
      "Cost after epoch 4050: 2.038142565491338\n",
      "Cost after epoch 4060: 2.0380633356514792\n",
      "Cost after epoch 4070: 2.0379841686044946\n",
      "Cost after epoch 4080: 2.037904912376793\n",
      "Cost after epoch 4090: 2.0378255778074394\n",
      "Cost after epoch 4100: 2.0377461757124546\n",
      "Cost after epoch 4110: 2.0376681606119873\n",
      "Cost after epoch 4120: 2.0375902612362413\n",
      "Cost after epoch 4130: 2.037512327018799\n",
      "Cost after epoch 4140: 2.037434367719523\n",
      "Cost after epoch 4150: 2.0373563930656386\n",
      "Cost after epoch 4160: 2.0372798289362852\n",
      "Cost after epoch 4170: 2.0372034255873905\n",
      "Cost after epoch 4180: 2.037127034389063\n",
      "Cost after epoch 4190: 2.0370506640686203\n",
      "Cost after epoch 4200: 2.03697432331529\n",
      "Cost after epoch 4210: 2.0368994059127945\n",
      "Cost after epoch 4220: 2.0368246870886257\n",
      "Cost after epoch 4230: 2.0367500206103846\n",
      "Cost after epoch 4240: 2.0366754142332044\n",
      "Cost after epoch 4250: 2.036600875671903\n",
      "Cost after epoch 4260: 2.036527763843397\n",
      "Cost after epoch 4270: 2.0364548817842647\n",
      "Cost after epoch 4280: 2.0363820860546245\n",
      "Cost after epoch 4290: 2.036309383506772\n",
      "Cost after epoch 4300: 2.0362367809529434\n",
      "Cost after epoch 4310: 2.03616560028338\n",
      "Cost after epoch 4320: 2.0360946746276585\n",
      "Cost after epoch 4330: 2.0360238636709034\n",
      "Cost after epoch 4340: 2.0359531734370577\n",
      "Cost after epoch 4350: 2.035882609912075\n",
      "Cost after epoch 4360: 2.0358134563412222\n",
      "Cost after epoch 4370: 2.0357445777329426\n",
      "Cost after epoch 4380: 2.0356758371714676\n",
      "Cost after epoch 4390: 2.0356072399288796\n",
      "Cost after epoch 4400: 2.0355387912426055\n",
      "Cost after epoch 4410: 2.0354717345691884\n",
      "Cost after epoch 4420: 2.0354049681386455\n",
      "Cost after epoch 4430: 2.0353383586546276\n",
      "Cost after epoch 4440: 2.035271910713394\n",
      "Cost after epoch 4450: 2.035205628880695\n",
      "Cost after epoch 4460: 2.0351407160920756\n",
      "Cost after epoch 4470: 2.0350761047516603\n",
      "Cost after epoch 4480: 2.0350116653538857\n",
      "Cost after epoch 4490: 2.0349474018928895\n",
      "Cost after epoch 4500: 2.034883318336906\n",
      "Cost after epoch 4510: 2.0348205767212075\n",
      "Cost after epoch 4520: 2.0347581442366103\n",
      "Cost after epoch 4530: 2.034695895295664\n",
      "Cost after epoch 4540: 2.0346338333603797\n",
      "Cost after epoch 4550: 2.0345719618716176\n",
      "Cost after epoch 4560: 2.0345114018846804\n",
      "Cost after epoch 4570: 2.0344511556965235\n",
      "Cost after epoch 4580: 2.0343911017259533\n",
      "Cost after epoch 4590: 2.03433124296816\n",
      "Cost after epoch 4600: 2.034271582401873\n",
      "Cost after epoch 4610: 2.034213200274054\n",
      "Cost after epoch 4620: 2.034155134057152\n",
      "Cost after epoch 4630: 2.0340972662298658\n",
      "Cost after epoch 4640: 2.034039599380574\n",
      "Cost after epoch 4650: 2.033982136085648\n",
      "Cost after epoch 4660: 2.0339259161636174\n",
      "Cost after epoch 4670: 2.03387001212076\n",
      "Cost after epoch 4680: 2.033814310519222\n",
      "Cost after epoch 4690: 2.03375881359495\n",
      "Cost after epoch 4700: 2.033703523575972\n",
      "Cost after epoch 4710: 2.0336494404025998\n",
      "Cost after epoch 4720: 2.0335956712972876\n",
      "Cost after epoch 4730: 2.033542106903251\n",
      "Cost after epoch 4740: 2.033488749152632\n",
      "Cost after epoch 4750: 2.0334355999733162\n",
      "Cost after epoch 4760: 2.0333836201147677\n",
      "Cost after epoch 4770: 2.033331951053062\n",
      "Cost after epoch 4780: 2.033280487487959\n",
      "Cost after epoch 4790: 2.0332292310907616\n",
      "Cost after epoch 4800: 2.0331781835316907\n",
      "Cost after epoch 4810: 2.033128267165292\n",
      "Cost after epoch 4820: 2.0330786571410533\n",
      "Cost after epoch 4830: 2.033029252172077\n",
      "Cost after epoch 4840: 2.032980053706332\n",
      "Cost after epoch 4850: 2.0329310631933697\n",
      "Cost after epoch 4860: 2.032883165472868\n",
      "Cost after epoch 4870: 2.032835568693547\n",
      "Cost after epoch 4880: 2.032788175523073\n",
      "Cost after epoch 4890: 2.032740987218429\n",
      "Cost after epoch 4900: 2.032694005040351\n",
      "Cost after epoch 4910: 2.0326480772571958\n",
      "Cost after epoch 4920: 2.032602444269094\n",
      "Cost after epoch 4930: 2.032557012626682\n",
      "Cost after epoch 4940: 2.0325117834235447\n",
      "Cost after epoch 4950: 2.032466757758695\n",
      "Cost after epoch 4960: 2.0324227483190103\n",
      "Cost after epoch 4970: 2.0323790269518507\n",
      "Cost after epoch 4980: 2.0323355040088518\n",
      "Cost after epoch 4990: 2.0322921804434446\n",
      "training time: 0:00:01.075698\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAH6lJREFUeJzt3Xt0nHd95/H3d2YkWXdZlhzfZFtO4tjOxU6i3EMusJCSAuGSLJxCNhTYLF1OmxzS3QJl2VJgu9DdlNIAOSkBSpuWQhPSEJpCljhACHFiO3Z8ixM7drAdO5LvtiTrMvPdP55Ho7Ei2aPLzDOXz+ucOfNcfjP6/mR5PvPcfo+5OyIiIgCxqAsQEZHCoVAQEZE0hYKIiKQpFEREJE2hICIiaQoFERFJUyiIiEiaQkFERNIUCiIikpaIuoDxamlp8YULF0ZdhohIUVmzZs1+d289XbuiC4WFCxeyevXqqMsQESkqZvZqNu20+0hERNIUCiIikqZQEBGRNIWCiIikKRRERCRNoSAiImkKBRERSSubUNixv5u7f7aVJ7d2cqR3IOpyREQKUtFdvDZRG/Yc4Z6V20g5mMG5cxq45eI2bumYR01l2fwaREROydw96hrGpaOjwyd6RXN33yDrdx1m9auH+NnmfWzcc5TZjdP43++7gGsXn/bqbxGRomVma9y947TtyikURnp2x0H+x8MbebnzGF9493l88LIFU/K+IiKFJttQKJtjCqO5tL2ZH33iSq5d3MpnH97Iv2/cF3VJIiKRKutQAKipTPDND13MBXMb+ZMHX6Dz6ImoSxIRiUzZhwLAtIo4f/X+FZwYSPLFn2yJuhwRkcgoFEKLWuv46NXtPLL+NV7cdzTqckREIqFQyHD7NYuor0rwjZXboy5FRCQSCoUMTTWVvO/ieTy2cS/7j/dFXY6ISN4pFEb40OXzGUg6D63dHXUpIiJ5p1AY4ayZ9Syf18hPXtgbdSkiInmnUBjFDefNYv3uI+w53Bt1KSIieaVQGMXbz5sNwE91MZuIlBmFwijaW2ppb6nl19v2R12KiEheKRTGcOWZM1i14yCDyVTUpYiI5I1CYQxXntnC8b5BNuw5EnUpIiJ5k7NQMLM2M1tpZpvNbJOZ3TFKm+vM7IiZrQsfn8tVPeN1+aJmAH7zyoGIKxERyZ9c3l1mELjL3deaWT2wxswed/fNI9r9yt3fkcM6JmRGXRXtLbWs++3hqEsREcmbnG0puPted18bTh8DtgBzc/XzcmFFWxPrdh2m2O45ISIyUXk5pmBmC4ELgVWjrL7CzNab2WNmdm4+6snWirYmOo/1sfeIhtMWkfKQ81AwszrgQeBOdx85/OhaYIG7Lwf+Bnh4jPe43cxWm9nqrq6u3BacYUVbEwDrdmkXkoiUh5yGgplVEATCA+7+0Mj17n7U3Y+H0/8GVJhZyyjt7nP3DnfvaG3N372Ul85uoDIeY/1uhYKIlIdcnn1kwP3AFne/e4w2s8J2mNmlYT0Fc7pPZSLGotZatu47FnUpIiJ5kcuzj64CbgU2mNm6cNlngPkA7n4vcDPwB2Y2CPQCH/ACO6q7dHYDz+i0VBEpEzkLBXd/CrDTtLkHuCdXNUyFc2bV86Pn93CkZ4DGmoqoyxERySld0XwaS2bVA+gWnSJSFhQKp7FkVgMAL+q4goiUAYXCaZzRUEVjdYVCQUTKgkLhNMyMM1tr2bH/eNSliIjknEIhC4ta63ilqzvqMkREck6hkIX2llo6j/VxvG8w6lJERHJKoZCFM1trAdi5X1sLIlLaFApZaG+pA2B7l44riEhpUyhkYcGMGsxgh7YURKTEKRSyMK0iztymaoWCiJQ8hUKW2ltqdQaSiJQ8hUKWFrXUsmN/t+7CJiIlTaGQpfkzajneN8jhnoGoSxERyRmFQpbmN9cA8OrBnogrERHJHYVCloZC4bcKBREpYQqFLLU1VwOwS6EgIiVMoZClmsoELXVV/PaAQkFESpdCYRzmN1dr95GIlDSFwjjMb65RKIhISVMojMP85hr2HumlfzAVdSkiIjmhUBiHtuYaUg6vHe6NuhQRkZxQKIyDTksVkVKnUBiH+TMUCiJS2hQK43BG/TQq4zFdqyAiJUuhMA6xmDFPp6WKSAlTKIzT/OYaXtUFbCJSohQK47QgvFZBQ2iLSClSKIzT0BDahzSEtoiUIIXCOKWH0D6gu7CJSOnJWSiYWZuZrTSzzWa2yczuOEXbS8xs0MxuzlU9U2WBTksVkRKWyOF7DwJ3uftaM6sH1pjZ4+6+ObORmcWBLwM/y2EtU6ZtehgKOtgsIiUoZ1sK7r7X3deG08eALcDcUZr+IfAg0JmrWqZSdWWcmfVVugObiJSkvBxTMLOFwIXAqhHL5wLvAb6ZjzqmyoIZNdpSEJGSlPNQMLM6gi2BO9396IjVXwX+xN1POeyomd1uZqvNbHVXV1euSs3a/OZaHVMQkZKU01AwswqCQHjA3R8apUkH8H0z2wncDHzDzN49spG73+fuHe7e0dramsuSszK/uYZ9R09wYiAZdSkiIlMqZweazcyA+4Et7n73aG3cvT2j/XeBR9394VzVNFWGzkDadbCHs8+oj7gaEZGpk8uzj64CbgU2mNm6cNlngPkA7n5vDn92TmWOlqpQEJFSkrNQcPenABtH+w/nqpapNnwBm44riEhp0RXNEzCjtpLayrgONotIyVEoTICZMX+GzkASkdKjUJigBc01Gv9IREqOQmGCFsyoYdfBXpIpDaEtIqVDoTBBZ7bW0Z9M6dacIlJSFAoTdObMOgC2dR6PuBIRkamjUJigs8JQeFmhICIlRKEwQY3VFcysr9KWgoiUFIXCJJw1s45tXQoFESkdCoVJOGtmHds7j+OuM5BEpDQoFCbh7Jl1HO8b5PWjfVGXIiIyJRQKkzB0BtJLrx+LuBIRkamhUJiEpbMaANi8d+S9g0REipNCYRKm11Yyb3o1G/YciboUEZEpoVCYpPPnNrJRoSAiJUKhMEnnzW3k1QM9HOkdiLoUEZFJUyhM0nlzGwHY9Jq2FkSk+CkUJumCMBSe/+3hiCsREZk8hcIkTa+t5KyZdTy382DUpYiITJpCYQpc2t7Mmp2HdG8FESl6CoUpcFl7M8f6Btmi6xVEpMgpFKbAJQubAXh2h3YhiUhxUyhMgTlN1bQ1V/P09v1RlyIiMikKhSly7eJWnt5+gL7BZNSliIhMmEJhily3eCY9/Ume23Eo6lJERCZMoTBFrjxrBpXxGE9u7Yy6FBGRCVMoTJGaygSXtjfz5EtdUZciIjJhCoUpdN05rWzrPM7uQz1RlyIiMiFZhYKZ3ZLNsnJ3/ZKZADy5VVsLIlKcst1S+HSWy9LMrM3MVprZZjPbZGZ3jNLmJjN7wczWmdlqM7s6y3oK0qKWWtqaq3VcQUSKVuJUK83s7cCNwFwz+1rGqgZg8DTvPQjc5e5rzaweWGNmj7v75ow2PwcecXc3swuAHwBLxt2LAmFmXH/OTH64ejcnBpJMq4hHXZKIyLicbkvhNWA1cAJYk/F4BLjhVC90973uvjacPgZsAeaOaHPc3YcGDKoFin7woOvPmUnvQFJXN4tIUTrlloK7rwfWm9k/uvsAgJlNB9rcPesT8s1sIXAhsGqUde8B/gKYCfxu1pUXqMsXzaAqEWPl1k6uWdwadTkiIuOS7TGFx82swcyagbXA35rZX2XzQjOrAx4E7nT3N4wY5+4/cvclwLuBL4zxHreHxxxWd3UV9kHc6so4V5w5QwebRaQoZRsKjeEH+nuB77n7ZcBbTvciM6sgCIQH3P2hU7V1918Ci8ysZZR197l7h7t3tLYW/rfv68+ZyY793ezc3x11KSIi45JtKCTMbDbwH4FHs3mBmRlwP7DF3e8eo81ZYTvM7CKgCjiQZU0F67pzguD6hS5kE5Eik20o/DnwU2C7uz9nZouAl0/zmquAW4E3h6ecrjOzG83s42b28bDN+4CNZrYO+Drw/owDz0VrwYxa5jZV88wrRZ9vIlJmTnmgeYi7/xD4Ycb8KwQf6Kd6zVOAnabNl4EvZ1NDsblsUTNPbu3C3Qk3hkRECl62VzTPM7MfmVln+HjQzOblurhidnn7DA529/Ny5/GoSxERyVq2u4++Q3Btwpzw8eNwmYzhskXB3dhWaReSiBSRbEOh1d2/4+6D4eO7QOGfBhSh+c01zGqYxjO6iE1Eiki2oXDAzD5kZvHw8SFK4CyhXDIzLlvUzKpXDlICx85FpExkGwofITgddR+wF7gZ+HCOaioZl7Y3s/94H68e0FDaIlIcxnNK6m3u3uruMwlC4vO5K6s0rGhrAmD97sMRVyIikp1sQ+GCzLGO3P0gwVhGcgqLz6hnWkWM9buORF2KiEhWsg2FWDgQHgDhGEhZXeNQziriMc6b06gtBREpGtl+sP9f4DdmNnQB2y3Al3JTUmlZ3tbEPzzzKgPJFBVx3f1URApbVp9S7v49gsHwXg8f73X3v89lYaVieVsTfYMptu47FnUpIiKnlfUuoPCOaZtP21BOsmLe8MHm8+Y2RlyNiMipaX9GjrU1VzO9poL1u3RcQUQKn0Ihx8yMc+c0snnvG+4vJCJScBQKeXDunAZe2necgWQq6lJERE5JoZAHy+Y00J9Msb1LI6aKSGFTKOTBstkNAGzao11IIlLYFAp50N5SS1UipuMKIlLwFAp5kIjHWDK7gc2vKRREpLApFPJk2ewGNu89qmG0RaSgKRTyZNmcBo70DrDncG/UpYiIjEmhkCdDB5u1C0lECplCIU+Wzq7HDB1sFpGCplDIk5rKBO0ttdpSEJGCplDIo2WzG9ikUBCRAqZQyKNlcxrYc7iXIz0DUZciIjIqhUIeDR1s3rJPWwsiUpgUCnmkM5BEpNApFPKotb6KlrpKtugMJBEpUAqFPDIzloZXNouIFKKchYKZtZnZSjPbbGabzOyOUdp80MxeMLMNZva0mS3PVT2FYtnsBl5+XfdWEJHClMsthUHgLndfBlwOfMLMlo1oswO41t3PB74A3JfDegrC0tm6t4KIFK6chYK773X3teH0MWALMHdEm6fd/VA4+wwwL1f1FIplc8IzkLQLSUQKUF6OKZjZQuBCYNUpmn0UeCwf9URpUUstlYmYzkASkYKUyPUPMLM64EHgTncf9ZPQzK4nCIWrx1h/O3A7wPz583NUaX4k4jHOOaOeLXuPRV2KiMgb5HRLwcwqCALhAXd/aIw2FwDfAm5y9wOjtXH3+9y9w907Wltbc1dwniydXa97K4hIQcrl2UcG3A9scfe7x2gzH3gIuNXdX8pVLYVm2ewGDnb303msL+pSREROksvdR1cBtwIbzGxduOwzwHwAd78X+BwwA/hGkCEMuntHDmsqCEszrmw+o2FaxNWIiAzLWSi4+1OAnabNx4CP5aqGQrU0PANp896jXL9kZsTViIgM0xXNEWiYVsG86dW6sllECo5CISLLZjfoWgURKTgKhYgsnd3Ajv3d9PQPRl2KiEiaQiEiy+Y04A5b9+l6BREpHAqFiKTvraBdSCJSQBQKEZk3vZr6qoSOK4hIQVEoRCR9bwWNgSQiBUShEKFlcxp4cd8xkikNdyEihUGhEKEL5jXS059kW6furSAihUGhEKHlbU0ArNt16DQtRUTyQ6EQofYZtTRMS7Bu15GoSxERARQKkYrFjOVtTazfdTjqUkREAIVC5JbPa2Lr68fo7U9GXYqIiEIhasvbmkimnE2vaReSiERPoRCx5W2NAKzTLiQRKQAKhYjNrJ/G3KZqhYKIFASFQgFY3tbI+t0KBRGJnkKhAKxoa2LXwV46j52IuhQRKXMKhQJwafsMAJ7dcTDiSkSk3CkUCsC5cxqoqYwrFEQkcgqFAlARj3HxgukKBRGJnEKhQFy6sJkX9x3jcE9/1KWISBlTKBSIS9ubAXhupwbHE5HoKBQKxPK2JirjMVa9ciDqUkSkjCkUCsS0ijgXLWji19sVCiISHYVCAbl28Uy27D3K60d1vYKIREOhUECuXdwKwC9f6oq4EhEpVwqFArJ0dj2t9VX8QqEgIhFRKBQQM+Paxa386uX9JFMedTkiUoZyFgpm1mZmK81ss5ltMrM7RmmzxMx+Y2Z9ZvbHuaqlmFy7uJUjvQMaIE9EIpHLLYVB4C53XwZcDnzCzJaNaHMQ+CPg/+SwjqLyprNbSMSMxze/HnUpIlKGchYK7r7X3deG08eALcDcEW063f05YCBXdRSbpppKrjhzBo9t2Iu7diGJSH7l5ZiCmS0ELgRW5ePnFbsbz5/NzgM9bN57NOpSRKTM5DwUzKwOeBC4090n9ClnZreb2WozW93VVfpn5txw7iziMeOxDfuiLkVEykxOQ8HMKggC4QF3f2ii7+Pu97l7h7t3tLa2Tl2BBaq5tpLLFzXz4xde0y4kEcmrXJ59ZMD9wBZ3vztXP6dUvffCebx6oEcD5IlIXuVyS+Eq4FbgzWa2LnzcaGYfN7OPA5jZLDPbDXwS+KyZ7TazhhzWVDTefv4s6qoS/GD1rqhLEZEyksjVG7v7U4Cdps0+YF6uaihmNZUJ3rl8Ng8//xp/9q5zqavK2T+ViEiarmguYLd0tNE7kORf1+2JuhQRKRMKhQJ2YVsT589t5P6ndpDSsBcikgcKhQJmZvznaxbxSlc3K7d2Rl2OiJQBhUKBu/G8WcxtqubeX2zX6akiknMKhQKXiMf4L9cu4rmdh/jly/ujLkdESpxCoQh84JL5tDVX85V/f1HHFkQkpxQKRaAyEeOut57DpteO8sj616IuR0RKmEKhSLxr+RyWz2vkiz/ZzJEeDSorIrmhUCgSsZjxv957Pod6BvjSv22OuhwRKVEKhSJy7pxGbr9mET9YvZtHX9BuJBGZegqFIvPJty7movlNfOrBDbzSdTzqckSkxCgUikxFPMY9v3cRFXHjI999jq5jfVGXJCIlRKFQhOY0VfOt2y7h9aN93PbtZznY3R91SSJSIhQKReriBdP55ocuYnvXcW7+5tPsOtgTdUkiUgIUCkXsunNm8g8fu4z9x/t45z1P8bNNun2niEyOFdt4Oh0dHb569eqoyygor3Qd54++/zwb9xzlHRfM5tM3LmVuU3XUZck4uTvJlDOYGvmcIpWCwVTq5OXJ4fVDy1MpJ+UQi0HMjJgZ8VgwuGLMjLgZZuG6GMTNSMRjVCZiVCWC58p48IjFTnk7FCkyZrbG3TtO206hUBr6BpN8Y+V27v3FdgDec+FcPnzVQpbMivZGdn2DSQ51D3Cwu59DPf0c7O6nu2+QgWSKgaQzkEwxmPJwPkXmKB5DH0lmQ/M2Yj4w9BJ38HAumA6egzbBguG2nm6T+dpUykm6k0xBMpUimYKUD3/gpj98ww/w9CNj/g3rRnzAp+eT4XNG+0JSEbcgIBLDj6pEnKpEjJrKONWVCWoq4tRUxqmpilNTmaB6aH5ofWWc6sp42C4RtgumayvjJOLaWZEvCoUytedwL/c8sY2H1u6mbzDF4jPqeNuyWVzS3syKeU001lRM+L0HkikO9wykP9wPdfdzsCd87s5YnrG+uz+Z9fvHY8E3WTj5wz2YD5/DBZkf5maZAWInhYkxvNIylzH8Ogt/phFcJBgfeljGdMyIGSRisbANxGMx4ka4zkjEh76ZG4mMZfFYjERseHl6fXp+xPr4yctjI14XPMdOah+34XVmRsqDcEqFofbGedLhNZhK0TeQoj+Zon8wRd9g8Dw0HyxLppedGEjR0z9Ib3+SnvDRO5Ckp3+QEwOpcf1NVSZi1IYhEYRL4qT52qrhABlaVz1ifvi1cWrDYNJWzhspFMrcoe5+Hl63h59u2sezOw6mv4Gf0VDFvOk1zGmqpq4qQV1VnOqKOEkf/vba3Z/kaO8AR08McKQ3eBzq7ufoicExf15dVYLptRU011QyvbaS6TXBo7m2gum1lenlzbWV1FUlqIjHqIgbFfEYibhREdPuilKQSnkYEMkgNAYGh6f7g+AYCpKevkG6+5P09gfPPf2DdPcNP/cOJOnuC9p39w8yno+qmsywqIxTWxWGzIgAqalMpIPnDW2rgoAZmq9KxNJfIIqRQkHSjp0YYMPuIzy/6zA793ez61AP+46c4HjfIMf7gm93Q99I42bUViVoqE7QWF1Bw7QKGqsraK4d+0O+qaaCqkQ86m5KCXP39BbKUEicHCAnz2eGT2aw9PSFQdUXzI9nyyYeszcES+ZWTE0YINMqggCpqhje3RbMZ0wn4uH6jDYj2k91AGUbCrobfBmon1bBlWe1cOVZLVGXIjIhZkZ1eHxixhS+bzLl6V1h3SMCpHeUQOnOaBts6QxyoLufXYd66Qm/ZPWFu+AmqzIxIjQSMX7vsvl87E2LpqDnY1MoiEjZiseM+mkV1E+b+LG20bg7/ckgHPoGgmMyb5geTHFiYGj58LK+wWTY7uTXnRhM0lJXNaV1jkahICIyxcws/IYfh2lRVzM+Oh9MRETSFAoiIpKmUBARkTSFgoiIpCkUREQkTaEgIiJpCgUREUlTKIiISFrRjX1kZl3AqxN8eQuwfwrLKQbqc3lQn8vDZPq8wN1bT9eo6EJhMsxsdTYDQpUS9bk8qM/lIR991u4jERFJUyiIiEhauYXCfVEXEAH1uTyoz+Uh530uq2MKIiJyauW2pSAiIqdQNqFgZr9jZlvNbJuZfSrqeibDzL5tZp1mtjFjWbOZPW5mL4fP08PlZmZfC/v9gpldlPGa28L2L5vZbVH0JRtm1mZmK81ss5ltMrM7wuWl3OdpZvasma0P+/z5cHm7ma0K+/bPZlYZLq8K57eF6xdmvNenw+VbzeyGaHqUPTOLm9nzZvZoOF/SfTaznWa2wczWmdnqcFl0f9vuXvIPIA5sBxYBlcB6YFnUdU2iP9cAFwEbM5Z9BfhUOP0p4Mvh9I3AY4ABlwOrwuXNwCvh8/RwenrUfRujv7OBi8LpeuAlYFmJ99mAunC6AlgV9uUHwAfC5fcCfxBO/1fg3nD6A8A/h9PLwr/3KqA9/H8Qj7p/p+n7J4F/BB4N50u6z8BOoGXEssj+tstlS+FSYJu7v+Lu/cD3gZsirmnC3P2XwMERi28C/i6c/jvg3RnLv+eBZ4AmM5sN3AA87u4H3f0Q8DjwO7mvfvzcfa+7rw2njwFbgLmUdp/d3Y+HsxXhw4E3A/8SLh/Z56Hfxb8Ab7Hgzu83Ad939z533wFsI/j/UJDMbB7wu8C3wnmjxPs8hsj+tsslFOYCuzLmd4fLSskZ7r43nN4HnBFOj9X3ovydhLsILiT45lzSfQ53o6wDOgn+k28HDrv7YNgks/5038L1R4AZFFmfga8C/x1IhfMzKP0+O/AzM1tjZreHyyL729Y9mkuQu7uZldxpZWZWBzwI3OnuR4MvhYFS7LO7J4EVZtYE/AhYEnFJOWVm7wA63X2NmV0XdT15dLW77zGzmcDjZvZi5sp8/22Xy5bCHqAtY35euKyUvB5uRhI+d4bLx+p7Uf1OzKyCIBAecPeHwsUl3ech7n4YWAlcQbC7YOjLXGb96b6F6xuBAxRXn68C3mVmOwl28b4Z+GtKu8+4+57wuZMg/C8lwr/tcgmF54Czw7MYKgkOSj0ScU1T7RFg6IyD24B/zVj+n8KzFi4HjoSbpT8F3mZm08MzG94WLis44X7i+4Et7n53xqpS7nNruIWAmVUDbyU4lrISuDlsNrLPQ7+Lm4EnPDgC+QjwgfBMnXbgbODZ/PRifNz90+4+z90XEvwffcLdP0gJ99nMas2sfmia4G9yI1H+bUd95D1fD4Kj9i8R7Jf906jrmWRf/gnYCwwQ7Dv8KMG+1J8DLwP/D2gO2xrw9bDfG4COjPf5CMFBuG3A70fdr1P092qC/a4vAOvCx40l3ucLgOfDPm8EPhcuX0TwAbcN+CFQFS6fFs5vC9cvynivPw1/F1uBt0fdtyz7fx3DZx+VbJ/Dvq0PH5uGPpui/NvWFc0iIpJWLruPREQkCwoFERFJUyiIiEiaQkFERNIUCiIikqZQkJJnZn9hZteb2bvN7NPjfG1rOALn82b2plzVOMbPPn76ViJTS6Eg5eAy4BngWuCX43ztW4AN7n6hu/9qyisTKTAKBSlZZvaXZvYCcAnwG+BjwDfN7HOjtF1oZk+EY9T/3Mzmm9kKgiGMbwrHuq8e8ZqLzewX4UBmP80YluBJM/vr8DUbzezScHmzmT0c/oxnzOyCcHmdmX3HgjH1XzCz92X8jC9ZcE+FZ8zsjHDZLeH7rjez8YacyKlFfUWfHnrk8kEQCH9DMPT0r0/R7sfAbeH0R4CHw+kPA/eM0r4CeBpoDeffD3w7nH4S+Ntw+hrC+16EdfzPcPrNwLpw+svAVzPee3r47MA7w+mvAJ8NpzcAc8Pppqh/x3qU1kOjpEqpu4hgCIElBGMHjeUK4L3h9N8TfAifyjnAeQSjWkJwI6e9Gev/CYJ7X5hZQziO0dXA+8LlT5jZDDNrAP4DwVg/hOsOhZP9wKPh9BqC8Y8Afg1818x+AAwNDigyJRQKUpLCXT/fJRgtcj9QEyy2dcAV7t472R8BbHL3K8ZYP3L8mImMJzPg7kOvSxL+f3X3j5vZZQQ3o1ljZhe7+4EJvL/IG+iYgpQkd1/n7isYvnXnE8AN7r5ijEB4muFv6x8ETndQeSvQamZXQDC0t5mdm7H+/eHyqwlGsjwSvucHw+XXAfvd/SjBDXQ+MfTCcJTLMZnZme6+yt0/B3Rx8pDJIpOiLQUpWWbWChxy95SZLXH3zado/ofAd8zsvxF80P7+qd7b3fvN7Gbga2bWSPB/6asEI10CnDCz5wmOPXwkXPZnwLfDg989DA+N/EXg62a2kWCL4POcerfQX5rZ2QRbKz8n2D0mMiU0SqrIFDOzJ4E/dvfVUdciMl7afSQiImnaUhARkTRtKYiISJpCQURE0hQKIiKSplAQEZE0hYKIiKQpFEREJO3/A7GzwwgUEi3mAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1115e8080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "paras = skipgram_model_training(X, Y_one_hot, vocab_size, 50, 0.05, 5000, batch_size=128, parameters=None, print_cost=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = np.arange(vocab_size)\n",
    "X_test = np.expand_dims(X_test, axis=0)\n",
    "softmax_test, _ = forward_propagation(X_test, paras)\n",
    "top_sorted_inds = np.argsort(softmax_test, axis=0)[-4:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loser's's neighbor words: ['game', 'market', \"loser's\", 'stock']\n",
      "investing's neighbor words: ['the', 'stock', 'beating', \"loser's\"]\n",
      "after's neighbor words: ['of', 'the', 'deduction', 'costs']\n",
      "deduction's neighbor words: ['costs', 'after', 'the', 'of']\n",
      "beating's neighbor words: ['market', 'stock', 'investing', 'costs']\n",
      "market's neighbor words: ['is', \"loser's\", 'a', 'beating']\n",
      "is's neighbor words: ['game', 'market', \"loser's\", 'stock']\n",
      "a's neighbor words: ['game', 'market', \"loser's\", 'stock']\n",
      "of's neighbor words: ['the', 'of', 'costs', 'beating']\n",
      "the's neighbor words: ['is', 'of', 'beating', 'stock']\n",
      "stock's neighbor words: ['investing', 'a', 'is', 'beating']\n",
      "game's neighbor words: ['is', \"loser's\", 'a', 'beating']\n",
      "costs's neighbor words: ['beating', 'of', 'deduction', 'the']\n"
     ]
    }
   ],
   "source": [
    "for input_ind in range(vocab_size):\n",
    "    input_word = id_to_word[input_ind]\n",
    "    output_words = [id_to_word[output_ind] for output_ind in top_sorted_inds[::-1, input_ind]]\n",
    "    print(\"{}'s neighbor words: {}\".format(input_word, output_words))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}