word2vec_SGNS_git.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#WORD2VEC SKIP-GRAM NEGATIVE SAMPLING REAL  WORLD EXAMPLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Make a sighle list of collections of strigs of partial sentences\n",
    "#dependencies\n",
    "import numpy as np\n",
    "import random\n",
    "import csv\n",
    "import re\n",
    "path = 'C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\World_Order_by_Henry_Kissinger.csv'\n",
    "\n",
    "#read the book\n",
    "def read_book(path):\n",
    "    f = open(path, encoding=\"utf8\")\n",
    "    csv_f = csv.reader(f)\n",
    "\n",
    "    book = []\n",
    "    for row in csv_f:\n",
    "        book.append(row[0])\n",
    "    \n",
    "    return book\n",
    "\n",
    "#processing get desired corpus, finding uniq set of words\n",
    "def preprocess(book):\n",
    "    \n",
    "    #join the string into a long single string\n",
    "    whole_book=' '.join(book)\n",
    "\n",
    "    #lower case and remove all the symbols\n",
    "    #then remove the digits\n",
    "    words_no_dig_punc = (re.sub(r'[^\\w]', ' ', whole_book.lower())).split()\n",
    "    words_no_dig_punc = [x for x in words_no_dig_punc if not any(c.isdigit() for c in x)]\n",
    "\n",
    "    #finding unique words\n",
    "    #Count and find most common words\n",
    "    from collections import Counter\n",
    "    word_counts = Counter(words_no_dig_punc)\n",
    "    word_counts = word_counts.most_common()   #A sorted version\n",
    "\n",
    "    #Top 20 most common are selected --> add and subtract few words to it as deemed appropriate\n",
    "    stop_words = [item[0] for item in (word_counts[:20])]\n",
    "    not_stop = ['world', 'order', 'war', 'not']\n",
    "    stop_words = [e for e in stop_words if e not in (not_stop)]\n",
    "    stop_words.append('chapter')\n",
    "    \n",
    "    #Make a corpus without these stop words\n",
    "    vocab = list(filter(lambda x: x not in stop_words, words_no_dig_punc))\n",
    "    \n",
    "    #From corpus get all uniq words --> to be indexed and tokenized (to one hot vectors)\n",
    "    uniq_words = list(set(vocab))\n",
    "    \n",
    "    words_to_ints ={k: v for v, k in enumerate(uniq_words)}  #Redundant, Not using it\n",
    "    ints_to_words ={v: k for v, k in enumerate(uniq_words)}  #Redundant, Not using it\n",
    "\n",
    "    #To tokenize all the words in corpus the indices of words in uniq_words work as look up table\n",
    "    vocab_int_pair= []\n",
    "    for i in range(len(vocab)):\n",
    "        vocab_int_pair.append([vocab[i], uniq_words.index(vocab[i])])\n",
    "\n",
    "    #Finally just take the tokenized version of corpus to be loaded into network to train    \n",
    "    int_arr_of_vocab = np.array(vocab_int_pair)[:, 1].astype(np.int)\n",
    "    \n",
    "    return (whole_book, vocab, uniq_words, words_to_ints,  vocab_int_pair, ints_to_words, int_arr_of_vocab)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "book = read_book(path)\n",
    "whole_book, vocab, uniq_words, words_to_ints,  vocab_int_pair, ints_to_words, int_arr_of_vocab =preprocess(book[:100]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(303, 437)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(uniq_words), len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###############################################################################################\n",
    "#Generating pairs of  true samples -->co-occuring within a given frame of window size, labeling the pair 1\n",
    "#int_arr_of_vocab --> Tokenized corpus\n",
    "#window --> Selected window\n",
    "#True pairs start at index +w and ends at index -w (length - w), as only these have full complement of pairs\n",
    "#Much easier to code this way than accomodating all elements for all window sizes\n",
    "\n",
    "def gen_true(int_arr_of_vocab, window):\n",
    "    \n",
    "    true_list = []\n",
    "    \n",
    "    #temp list initially collects 11 indices, however middle one is deleted\n",
    "    #On the second go, middle index is combined with all 10 in temp seperately and labeled as 1\n",
    "    for i in range(len(int_arr_of_vocab)-window*2):\n",
    "        tempp = []\n",
    "        tempp=list(int_arr_of_vocab[i:i+ (window *2) + 1 ]) \n",
    "        tempp.remove(int_arr_of_vocab[i+window])\n",
    "\n",
    "        for j in range(window*2):\n",
    "            true_list.append([int_arr_of_vocab[i+window], tempp[j], 1])\n",
    "    return true_list\n",
    "###################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###############################################################################################\n",
    "\n",
    "#int_arr_of_vocab --> Tokenized corpus\n",
    "#window --> Selected window\n",
    "#k --> is number of negative samples for each word --> keep 20 as each word as 10 positive pairs when window is size  10\n",
    "#Simply take k random samples from whole set of uniq_words and pair with each input word\n",
    "#Note that each input has window *2 true pairs and k has to be proportionately large\n",
    "\n",
    "#Alternate version, not used here\n",
    "#speed it up --> draw enough random samples in a range\n",
    "#concatenate --> the each item in true copied 20 x, random samples, 0s\n",
    "\n",
    "\n",
    "import random\n",
    "def gen_false(int_arr_of_vocab, uniq_words, k, window):\n",
    "    false_pairs = []\n",
    "    for i in range(len(int_arr_of_vocab)):\n",
    "        rnd_indices = random.sample(range(len(uniq_words)),  k)\n",
    "        for j in range(k):\n",
    "                false_pairs.append([int_arr_of_vocab[i], rnd_indices[j], 0])\n",
    "    \n",
    "    return false_pairs[k*window:-k*window]\n",
    "\n",
    "#Remove the first few pairs and last few pairs as true pairs start at index +w and ends at index -w (length - w)\n",
    "##############################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_list = gen_true(int_arr_of_vocab, window=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([['reiler', 83],\n",
       "  ['lion', 196],\n",
       "  ['on', 143],\n",
       "  ['cllaracler', 227],\n",
       "  ['course', 133],\n",
       "  ['history', 194],\n",
       "  ['henry', 159],\n",
       "  ['kissinger', 119],\n",
       "  ['world', 86],\n",
       "  ['order', 31],\n",
       "  ['reflections', 249]],\n",
       " [[194, 83, 1],\n",
       "  [194, 196, 1],\n",
       "  [194, 143, 1],\n",
       "  [194, 227, 1],\n",
       "  [194, 133, 1],\n",
       "  [194, 159, 1],\n",
       "  [194, 119, 1],\n",
       "  [194, 86, 1],\n",
       "  [194, 31, 1],\n",
       "  [194, 249, 1],\n",
       "  [159, 196, 1]],\n",
       " 4270)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_int_pair[:11], true_list[:11], len(true_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "false_list = gen_false(int_arr_of_vocab, uniq_words, k = 100, window = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([[194, 2, 0],\n",
       "  [194, 282, 0],\n",
       "  [194, 200, 0],\n",
       "  [194, 226, 0],\n",
       "  [194, 47, 0],\n",
       "  [194, 185, 0],\n",
       "  [194, 125, 0],\n",
       "  [194, 132, 0],\n",
       "  [194, 262, 0],\n",
       "  [194, 241, 0],\n",
       "  [159, 230, 0]],\n",
       " 42700)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "false_list[90:101], len(false_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###########################################################################################################\n",
    "#Concatenate true_list, false_list\n",
    "#False list keeps changing each time joint list is drawn\n",
    "def gen_joint_list(true_list,int_arr_of_vocab, uniq_words, k, window ):\n",
    "    joint_list = np.concatenate((np.array(true_list), np.array(gen_false(int_arr_of_vocab, uniq_words, k, window))), axis = 0)\n",
    "    np.random.shuffle(joint_list)\n",
    "    return joint_list\n",
    "############################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(883410, array([[1939, 2097,    1],\n",
       "        [ 912, 1947,    0],\n",
       "        [ 566, 1502,    0],\n",
       "        [2017,  834,    0],\n",
       "        [1483, 2109,    0]]))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(joint_list), joint_list[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#########################################################################################\n",
    "#As joint list is too long and takes a lot of memory to process at one go --> load small batches\n",
    "# i --> is used to return one batch at a time, it is a counter and a markers for selecting size\n",
    "#len(joint_list)//batch_size +1 --> gives the total numbers of batches\n",
    "\n",
    "def gen_batch(joint_list, batch_size, i):\n",
    "\n",
    "    if i < len(joint_list)//batch_size:\n",
    "        \n",
    "        batch = joint_list[i*batch_size:i*batch_size+batch_size]\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        batch = joint_list[i*batch_size:]\n",
    "        \n",
    "    return batch\n",
    "############################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[129,  29,   0],\n",
       "       [143,  68,   0],\n",
       "       [ 31, 255,   0],\n",
       "       [261, 192,   0],\n",
       "       [204, 218,   0]])"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Test batch b1\n",
    "b1 = gen_batch(joint_list, batch_size = 100, i =0)\n",
    "b1[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###########################################################################################################\n",
    "import torch\n",
    "\n",
    "#generate tensors of one hot vector for each tokenized pair in a batch of joint list\n",
    "#Also note labels are simply 3rd column in each batch\n",
    "\n",
    "def one_hot_auto_batchwise(batch, uniq_words):\n",
    "    \n",
    "    iol_tensor = torch.Tensor(batch).long()\n",
    "    \n",
    "    \n",
    "    middle_word_arr = torch.zeros(iol_tensor.shape[0], len(uniq_words))\n",
    "    sur_word_arr = torch.zeros(iol_tensor.shape[0], len(uniq_words))\n",
    "    for i in range(len(iol_tensor)):\n",
    "        middle_word_arr[i, iol_tensor[i, 0]] = 1\n",
    "        sur_word_arr[i, iol_tensor[i, 1]] = 1\n",
    "    labels = iol_tensor[:, 2].float()\n",
    "    return (middle_word_arr, sur_word_arr, labels)\n",
    "#################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#with test batch b1 get the respective vectors and labels\n",
    "mh, sh, ll = one_hot_auto_batchwise(b1, uniq_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(129), tensor(29), tensor(0.))"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check that these are consistent\n",
    "np.argmax(mh[0]), np.argmax(sh[0]), ll[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[129,  29,   0],\n",
       "       [143,  68,   0],\n",
       "       [ 31, 255,   0],\n",
       "       [261, 192,   0],\n",
       "       [204, 218,   0]])"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Tally with the joint list\n",
    "joint_list[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Defining the network\n",
    "#2 linear fully connected layers are used, bias units are not use\n",
    "#fc_midl_word takes all the input words/tokens\n",
    "#fc_sur_word takes all the targets (true or false counterpart) of the pair\n",
    "#Using sigmoid activation hence BCE loss\n",
    "#Also note parameters of both networks are combined in a list which helps in back prop and stepping\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "embed_size = 30\n",
    "def gen_model(uniq_words, embed_size, LR=0.0001):\n",
    "    \n",
    "\n",
    "    fc_midl_word = nn.Linear(len(uniq_words), embed_size, bias = False)\n",
    "    fc_sur_word = nn.Linear(len(uniq_words), embed_size, bias = False)\n",
    "\n",
    "    fc_midl_word = fc_midl_word.to(device)\n",
    "    fc_sur_word =fc_sur_word.to(device)\n",
    "\n",
    "    \n",
    "    criterion = nn.BCELoss()\n",
    "    \n",
    "    params = list(fc_midl_word.parameters()) + list(fc_sur_word.parameters())\n",
    "    optimizer = optim.Adam(params, lr = LR)\n",
    "    \n",
    "    return(fc_midl_word, fc_sur_word, criterion, optimizer )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fc_midl_word, fc_sur_word, criterion, optimizer = gen_model(uniq_words, embed_size =30, LR=0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Linear(in_features=303, out_features=30, bias=False),\n",
       " Linear(in_features=303, out_features=30, bias=False))"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fc_midl_word, fc_sur_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-0.1138,  0.0950, -0.0161,  ..., -0.0832, -0.0046,  0.0394],\n",
       "        [-0.0093, -0.0034,  0.0303,  ..., -0.0456, -0.1076, -0.0311],\n",
       "        [-0.0777,  0.0906,  0.1224,  ...,  0.0390, -0.0212,  0.0251],\n",
       "        ...,\n",
       "        [ 0.0622,  0.1260, -0.0003,  ..., -0.0900, -0.0741,  0.0520],\n",
       "        [ 0.0216,  0.1335, -0.1219,  ..., -0.0514,  0.0699, -0.1048],\n",
       "        [ 0.0550,  0.0750, -0.0896,  ...,  0.1106,  0.0030, -0.1137]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Collect losses\n",
    "#k --> proporion of negative samples --> with window of 5, k of 100 --> 10 negative samples for each tre pair\n",
    "#window --> selected/defined window for cooccurence\n",
    "#LR - Learing rate\n",
    "#embed_size - Size of embedding for vector representation\n",
    "#embed_size --> Arbitrary, May be scaled down by some log when uniq_set is large or by root when uniq_set is not so large\n",
    "\n",
    "#Get all the variable and network ready before feeding/starting algorithm\n",
    "#xavier init is vital for better and faster convergence\n",
    "\n",
    "losses = []\n",
    "k = 100\n",
    "window = 5\n",
    "LR = 0.001\n",
    "embed_size =30\n",
    "whole_book, vocab, uniq_words, words_to_ints,  vocab_int_pair, ints_to_words, int_arr_of_vocab =preprocess(book[:100]) \n",
    "true_list = gen_true(int_arr_of_vocab, window =5)\n",
    "\n",
    "#false list is generated everytime joint list is called\n",
    "#false_list = gen_false(int_arr_of_vocab, uniq_words, k = 100, window = 5)\n",
    "\n",
    "fc_midl_word, fc_sur_word, criterion, optimizer = gen_model(uniq_words, embed_size, LR)\n",
    "torch.nn.init.xavier_uniform_(fc_midl_word.weight)\n",
    "torch.nn.init.xavier_uniform_(fc_sur_word.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6926180720329285\n",
      "0.6926180720329285 0.6928567886352539\n",
      "0.29256561398506165\n",
      "0.29256561398506165 0.6926180720329285\n",
      "0.27223339676856995\n",
      "0.27223339676856995 0.29256561398506165\n",
      "0.2556808292865753\n",
      "0.2556808292865753 0.27223339676856995\n",
      "0.2138306200504303\n",
      "0.2138306200504303 0.2556808292865753\n",
      "0.17514534294605255\n",
      "0.17514534294605255 0.2138306200504303\n",
      "0.13904179632663727\n",
      "0.13904179632663727 0.17514534294605255\n",
      "0.125886932015419\n",
      "0.125886932015419 0.13904179632663727\n",
      "0.11686218529939651\n",
      "0.11686218529939651 0.125886932015419\n",
      "0.11403784155845642\n",
      "0.11403784155845642 0.11686218529939651\n"
     ]
    }
   ],
   "source": [
    "#Implement \n",
    "\n",
    "\n",
    "epochs = 500\n",
    "print_every = 50\n",
    "batch_size = 10000\n",
    "\n",
    "#save files containing weights whenever losses are min\n",
    "save_path1 = 'C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\fc_midl_dict.pth'\n",
    "save_path2 = 'C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\fc_sur_dict.pth'\n",
    "save_path3 = 'C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\fc_midl_wt.pth'\n",
    "\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    #Get fresh joint list with different random false samples\n",
    "    joint_list = gen_joint_list(true_list,int_arr_of_vocab, uniq_words, k, window )\n",
    "    num_batches = (len(joint_list)//batch_size) +1\n",
    "    \n",
    "    #Get i.th batch from joint list and proceed forward, backward\n",
    "    for i in range(num_batches):  \n",
    "        \n",
    "        batch = gen_batch(joint_list, batch_size, i)\n",
    "        mid_word_oh, sur_word_oh, labels = one_hot_auto_batchwise(batch, uniq_words)\n",
    "    \n",
    "    \n",
    "        z_midl = fc_midl_word(torch.Tensor(mid_word_oh))\n",
    "        \n",
    "        z_sur = fc_sur_word(torch.Tensor(sur_word_oh))\n",
    "        \n",
    "        #vector product of word as input and word as target, not the product is parallelized and not looped\n",
    "        #after training product/score for true pairs will be high and low/neg for false pairs\n",
    "        dot_inp_tar = torch.sum(torch.mul(z_midl, z_sur), dim =1).reshape(-1, 1)\n",
    "        \n",
    "        #sigmoid activation squashes the scores to 1 or 0\n",
    "        sig_logits = nn.Sigmoid()(dot_inp_tar)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss = criterion(sig_logits, torch.Tensor(labels).view(sig_logits.shape[0], 1))\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        \n",
    "    if epoch % print_every == 0:\n",
    "\n",
    "        losses.append(loss.item())\n",
    "        print(loss.item())\n",
    "        \n",
    "        if loss < np.min(losses[:-1]):\n",
    "            print(loss.item(),  np.min(losses[:-1] ))\n",
    "            torch.save(fc_midl_word.state_dict(), save_path1)\n",
    "            torch.save(fc_sur_word.state_dict(), save_path2)\n",
    "            torch.save(fc_midl_word.weight, save_path3)\n",
    "        #mid_hot, sur_hot, labels = one_hot_auto()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#non vectorized version for computing sig_logits\n",
    "'''\n",
    "        dot_u_v = torch.zeros(mid_word_oh.shape[0], 1)\n",
    "        for j in range(len(z_midl)):\n",
    "           dot_u_v[j, :] = z_midl[j, :] @ z_sur[j, :]\n",
    "        desired_logits = dot_u_v\n",
    "        sig_logits = nn.Sigmoid()(desired_logits)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#################################################################################################################\n",
    "#Given the set of uniq_words used to train, the function finds the cosine distances from selected word to all words\n",
    "#top_n words are returned, sim_score is simply the cosine distance\n",
    "import torch\n",
    "def find_dist(uniq_words, word, top_n):\n",
    "    distances = []\n",
    "    idx =  uniq_words.index(word)\n",
    "    for i in range(fc_midl_word.weight.t().shape[0]):\n",
    "        dist = nn.CosineSimilarity(dim = 0)(fc_midl_word.weight.t()[idx, :], fc_midl_word.weight.t()[i, :])\n",
    "        distances.append(dist)\n",
    "    sim_score, indices = torch.topk(torch.Tensor(distances), top_n)\n",
    "    indices = indices.tolist()\n",
    "    similar_words = [uniq_words[i] for i  in indices]\n",
    "    #print(similar_words)\n",
    "    print(sim_score)\n",
    "    return similar_words\n",
    "\n",
    "\n",
    "\n",
    "#Use when saved weights are transfered to new network for evaluation\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def find_dist_new(model, uniq_words, word, top_n):\n",
    "    distances = []\n",
    "    idx =  uniq_words.index(word)\n",
    "    for i in range(model.weight.t().shape[0]):\n",
    "        dist = nn.CosineSimilarity(dim = 0)(model.weight.t()[idx, :], model.weight.t()[i, :])\n",
    "        distances.append(dist)\n",
    "    confidence, indices = torch.topk(torch.Tensor(distances), top_n)\n",
    "    indices = indices.tolist()\n",
    "    similar_words = [uniq_words[i] for i  in indices]\n",
    "    #print(similar_words)\n",
    "    print(confidence)\n",
    "    return similar_words\n",
    "###################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.5577,  1.3201,  0.3211, -0.6869, -1.3027,  0.1036, -0.1439,  0.0094,\n",
       "         0.2495,  0.2930,  0.7856, -0.3296,  0.0779,  0.0496,  0.1061,  1.3059,\n",
       "        -0.7851, -0.0888, -1.1694, -0.7286,  0.9889,  0.1502,  0.4138,  0.6048,\n",
       "        -1.1535, -0.5175, -0.1529,  0.1102,  0.1541,  1.4733],\n",
       "       grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#vector/embedding for the 1st word in uniq_words\n",
    "fc_midl_word.weight.t()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Loading the saved weights\n",
    "pretrained_weights10 = torch.load('C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\fc_midl_wt.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.5225,  1.3043,  0.3189, -0.6499, -1.3031,  0.1121, -0.1322,  0.0210,\n",
       "         0.2307,  0.2580,  0.7552, -0.3287,  0.0626,  0.0117,  0.1040,  1.2341,\n",
       "        -0.7493, -0.0819, -1.0798, -0.7071,  0.9599,  0.1318,  0.3808,  0.5605,\n",
       "        -1.0795, -0.4833, -0.1719,  0.0804,  0.1467,  1.4072],\n",
       "       grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrained_weights10.t()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Transferring the weights\n",
    "fc_x = nn.Linear(len(uniq_words), embed_size, bias = False)\n",
    "fc_x.weight = pretrained_weights10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0000, 0.9841, 0.9608, 0.9521, 0.9298])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['khomeini', 'statecraft', 'iranian', 'tradition', 'iran']"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Using saved weights for finding similar words\n",
    "find_dist_new(uniq_words, fc_x, 'khomeini', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0000, 0.9625, 0.9334, 0.9170, 0.8833])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['nuclear', 'proliferation', 'challenge', 'age', 'cyber']"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_dist(uniq_words, 'nuclear', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0000, 0.9593, 0.9162, 0.9065, 0.8865])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['nuclear', 'proliferation', 'age', 'challenge', 'cyber']"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_dist(uniq_words, 'nuclear', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0000, 0.9584, 0.9538, 0.9348, 0.9154])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['henry', 'history', 'kissinger', 'character', 'reflections']"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_dist(uniq_words, 'henry', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0000, 0.9766, 0.9453, 0.9426, 0.8780, 0.8698, 0.8566, 0.8455, 0.8369,\n",
      "        0.8232])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['khomeini',\n",
       " 'statecraft',\n",
       " 'iranian',\n",
       " 'tradition',\n",
       " 'iran',\n",
       " 'approaches',\n",
       " 'revolution',\n",
       " 'vision',\n",
       " 'proliferation',\n",
       " 'nuclear']"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_dist(uniq_words, 'khomeini', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0000, 0.9648, 0.9507, 0.8777, 0.8735, 0.8661, 0.8430, 0.8413, 0.8241,\n",
      "        0.8137])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['ronald',\n",
       " 'reagan',\n",
       " 'renewal',\n",
       " 'beginning',\n",
       " 'richard',\n",
       " 'nixon',\n",
       " 'cold',\n",
       " 'vietnam',\n",
       " 'breakdown',\n",
       " 'afghanistan']"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_dist(uniq_words, 'ronald', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x22a15b14710>]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgkAAAFkCAYAAACq4KjhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XmUXVWZ9/Hvk0ogCUM1kJABqmSSQQUkZUBEBhkSAUFp\nESyxQbR5RbHFqK3tGMS3m6Uo2g4RtJFBoFrsFkVfJAyitCIgKUGmgGCQhCEJg2FIgiHZ7x+7bqcq\nuZXUvTWcO3w/a51VdU+dc++TwyH1y9777B0pJSRJktY1qugCJElSbTIkSJKksgwJkiSpLEOCJEkq\ny5AgSZLKMiRIkqSyDAmSJKksQ4IkSSrLkCBJksoyJEiSpLKqCgkRcUZELIiIFRFxa0RM38CxF0XE\nmohY3fO1tN1dfdmSJGm4VRwSIuJE4KvAbGAf4C5gbkRM6OeUDwOTgSk9X7cHngGurKZgSZI0MqLS\nBZ4i4lbgtpTSmT2vA1gIfCOl9OUBnP824L+AHVNKCysvWZIkjYSKWhIiYgzQAdxY2pdyyrgB2H+A\nb/Ne4AYDgiRJtW10hcdPAFqAxevsXwzstrGTI2IKcCTwzo0ctw0wE3gEWFlhjZIkNbOxwA7A3JTS\n04N5o0pDwmC9B3gW+OlGjpsJXD7s1UiS1LhOAq4YzBtUGhKeAlYDk9bZPwl4cgDnnwpcmlJ6eSPH\nPQJw2WWXsccee1RYYvOaNWsWX/va14ouo+543SrnNauO161yXrPK3X///bz73e+Gnt+lg1FRSEgp\nrYqIecBhwNXwvwMXDwO+saFzI+IQYGfgwgF81EqAPfbYg2nTplVSYlNrbW31elXB61Y5r1l1vG6V\n85oNyqC766vpbjgPuLgnLNwOzALGAxcDRMQ5wNSU0inrnPc+8lMR91dfriRJGikVh4SU0pU9cyKc\nTe5muBOYmVJa2nPIZKCt9zkRsSVwHHnOBEmSVAeqGriYUpoDzOnnZ6eW2fccsHk1nyVJkopR02s3\nLF268WO0VmdnZ9El1CWvW+W8ZtXxulXOa1asimdcHAkRMQ2YN3v2PM46ywErkiQNVHd3Nx0dHQAd\nKaXuwbxXTbck3Hpr0RVIktS8ajok3HYbrFlTdBWSJDWnmg4Jf/0rdA+qoUSSJFWrpkPC+PFw3XVF\nVyFJUnOq6ZAwfTrMnVt0FZIkNaeaDgmvfz3ccgs891zRlUiS1HxqOiTsvz+8/DLcdFPRlUiS1Hxq\nOiS0tcHOOzsuQZKkItR0SACYMcNxCZIkFaHmQ8LMmfDww3mTJEkjp+ZDwpveBKNH2+UgSdJIq/mQ\nsOWWeQCjXQ6SJI2smg8JkLscfvlLWLWq6EokSWoedRMSnn/eBZ8kSRpJdRESpk2Dbbaxy0GSpJFU\nFyFh1Cg44ghDgiRJI6kuQgLkLod58+Cpp4quRJKk5lA3IWHGDEgJbrih6EokSWoOdRMSpk6F17zG\nLgdJkkZK3YQEyF0O112XWxQkSdLwqruQ8PjjcO+9RVciSVLjq6uQ8MY3wtixdjlIkjQS6iokjBsH\nBx9sSJAkaSTUVUiA3OVw882wYkXRlUiS1NjqLiTMmAEvvZSDgiRJGj51FxJe9SrYbju7HCRJGm51\nFxIi1j4KKUmShk/dhQTIXQ733guLFhVdiSRJjasuQ8Lhh+cWBVsTJEkaPnUZErbZBqZPNyRIkjSc\n6jIkQO5yuP56WL266EokSWpMdRsSZs6EZ57Jy0dLkqShV7chYb/9YMstfRRSkqThUrchYcwYOPRQ\nxyVIkjRc6jYkQO5y+N3vYNmyoiuRJKnx1H1IWL0afvnLoiuRJKnx1HVI2HFH2GUXuxwkSRoOdR0S\nILcmzJ0LKRVdiSRJjaUhQsKCBfDQQ0VXIklSY6kqJETEGRGxICJWRMStETF9I8dvEhH/GhGPRMTK\niPhzRLynqorX8aY35Scd7HKQJGloVRwSIuJE4KvAbGAf4C5gbkRM2MBpPwLeBJwK7Ap0Ag9UXG0Z\nm28Ob3iD8yVIkjTUqmlJmAVckFK6NKU0HzgdWA68t9zBEfFm4EDgqJTSTSmlR1NKt6WUfld11euY\nORNuugn+9rehekdJklRRSIiIMUAHcGNpX0opATcA+/dz2jHAHcAnI2JRRDwQEedGxNgqa17PzJnw\nwgtwyy1D9Y6SJKnSloQJQAuweJ39i4HJ/ZyzE7kl4dXA24AzgeOBb1f42f167Wth4kTHJUiSNJRG\nj8BnjALWAO9KKb0AEBEfBX4UER9MKb3U34mzZs2itbW1z77Ozk46Ozv7fsAoOOKIPC7h3/5tyOuX\nJKkmdXV10dXV1WffsiGchrjSkPAUsBqYtM7+ScCT/ZzzBPBYKSD0uB8IYHvg4f4+7Gtf+xrTpk0b\nUGEzZ8IVV8CSJbDttgM6RZKkulbuH87d3d10dHQMyftX1N2QUloFzAMOK+2LiOh53d+IgN8CUyNi\nfK99u5FbFxZVVO0GHHFE/nrDDUP1jpIkNbdqnm44DzgtIk6OiN2B84HxwMUAEXFORFzS6/grgKeB\niyJij4g4CPgycOGGuhoqNWUK7LWXj0JKkjRUKh6TkFK6smdOhLPJ3Qx3AjNTSkt7DpkMtPU6/sWI\nOAL4JvB7cmD4IfC5Qda+npkz4Qc/yFM0Rwz1u0uS1FyqGriYUpoDzOnnZ6eW2fcgMLOaz6rEjBlw\n7rlw9925VUGSJFWv7tdu6O2Nb4Rx4+xykCRpKDRUSBg7Fg45xJAgSdJQaKiQAHDAAfCHPxRdhSRJ\n9a/hQsIrXgHPPAMvvlh0JZIk1beGCwltPc9VLFxYbB2SJNW7hgsJ7e35qyFBkqTBabiQsN12eY6E\nRx8tuhJJkupbw4WETTaByZMNCZIkDVbDhQTIXQ52N0iSNDgNGRLa2mxJkCRpsBoyJLS3GxIkSRqs\nhg0JCxfmhZ4kSVJ1GjIktLXBypXw1FNFVyJJUv1qyJDgXAmSJA1eQ4aE0qyLjkuQJKl6DRkSJk6E\nTTc1JEiSNBgNGRJGjcqtCXY3SJJUvYYMCeBcCZIkDVbDhgRnXZQkaXAaOiTYkiBJUvUaNiS0tcHj\nj8OqVUVXIklSfWrYkNDenmdcfPzxoiuRJKk+NWxIcK4ESZIGx5AgSZLKatiQsMUWsNVWPuEgSVK1\nGjYkgHMlSJI0GA0dEpwrQZKk6jV8SLAlQZKk6jR0SLC7QZKk6jV0SGhvh7/+FZ5/vuhKJEmqPw0d\nEkqPQTouQZKkyjV0SGhvz1/tcpAkqXINHRKmToVRo2xJkCSpGg0dEsaMgSlTbEmQJKkaDR0SwLkS\nJEmqVlOEBFsSJEmqXMOHBOdKkCSpOg0fEtrbYdEiWLOm6EokSaovDR8S2trgpZdg6dKiK5Ekqb40\nfEgozZXg4EVJkipTVUiIiDMiYkFErIiIWyNi+gaOPTgi1qyzrY6Ibasve+CcUEmSpOpUHBIi4kTg\nq8BsYB/gLmBuREzYwGkJeCUwuWebklJaUnm5ldtmGxg71pAgSVKlqmlJmAVckFK6NKU0HzgdWA68\ndyPnLU0pLSltVXxuVSKcK0GSpGpUFBIiYgzQAdxY2pdSSsANwP4bOhW4MyIej4jrIuIN1RRbLedK\nkCSpcpW2JEwAWoDF6+xfTO5GKOcJ4P3A24G/BxYCv4qI11b42VVzrgRJkio3erg/IKX0IPBgr123\nRsTO5G6LU4b78yG3JFx77Uh8kiRJjaPSkPAUsBqYtM7+ScCTFbzP7cABGzto1qxZtLa29tnX2dlJ\nZ2dnBR+VWxKefBL+9jfYZJOKTpUkqWZ1dXXR1dXVZ9+yZcuG7P0jDymo4ISIW4HbUkpn9rwO4FHg\nGymlcwf4HtcBz6WUju/n59OAefPmzWPatGkV1VfO9dfDjBnw5z/DjjsO+u0kSapZ3d3ddHR0AHSk\nlLoH817VdDecB1wcEfPILQKzgPHAxQARcQ4wNaV0Ss/rM4EFwL3AWOA04E3AEYMpvBK950owJEiS\nNDAVh4SU0pU9cyKcTe5muBOYmVIqTXw8GWjrdcom5HkVppIflfwjcFhK6ebBFF6Jtp5qHLwoSdLA\nVTVwMaU0B5jTz89OXef1ucCAuiGGy/jxeVIl50qQJGngGn7thhLnSpAkqTJNExLa2mxJkCSpEk0T\nEmxJkCSpMoYESZJUVtOEhLY2eO45GMI5JiRJamhNExJKcyU4LkGSpIFpmpDgXAmSJFWmaULClCnQ\n0mJLgiRJA9U0IWH0aNhuO1sSJEkaqKYJCeBcCZIkVaKpQoKPQUqSNHCGBEmSVFZThYS2Nli0CNas\nKboSSZJqX1OFhPZ2WLUKFi8uuhJJkmpfU4UE50qQJGngmiokOOuiJEkD11QhYautYLPNbEmQJGkg\nmiokRDhXgiRJA9VUIQF8DFKSpIEyJEiSpLKaLiTY3SBJ0sA0XUhob8/zJLz0UtGVSJJU25ouJJTm\nSli0qNg6JEmqdU0XEkpzJTguQZKkDWu6kLD99vmrIUGSpA1rupAwbhxMnOjgRUmSNqbpQgL4GKQk\nSQNhSJAkSWU1ZUhwrgRJkjauKUNCqSUhpaIrkSSpdjVlSGhrgxdegGXLiq5EkqTa1ZQhwbkSJEna\nOEOCJEkqqylDwqRJMHq0gxclSdqQpgwJLS155kVbEiRJ6l9ThgRwrgRJkjamaUOCcyVIkrRhTRsS\nbEmQJGnDmjYktLXBY4/B6tVFVyJJUm1q2pDQ3g4vvwxPPll0JZIk1aamDglgl4MkSf2pKiRExBkR\nsSAiVkTErRExfYDnHRARqyKiu5rPHUptbfmrgxclSSqv4pAQEScCXwVmA/sAdwFzI2LCRs5rBS4B\nbqiiziHX2gpbbGFLgiRJ/ammJWEWcEFK6dKU0nzgdGA58N6NnHc+cDlwaxWfOeQicpeDLQmSJJVX\nUUiIiDFAB3BjaV9KKZFbB/bfwHmnAjsCX6iuzOHR1mZLgiRJ/am0JWEC0AIsXmf/YmByuRMi4pXA\nvwEnpZTWVFzhMHKuBEmS+jd6ON88IkaRuxhmp5QeLu0e6PmzZs2itbW1z77Ozk46OzuHpL62Nrjq\nqiF5K0mSRlxXVxddXV199i1btmzI3j9yb8EAD87dDcuBt6eUru61/2KgNaV03DrHtwLPAi+zNhyM\n6vn+ZWBGSulXZT5nGjBv3rx5TJs2rZI/T0UuvRROOQWWL4dx44btYyRJGjHd3d10dHQAdKSUBvU0\nYUXdDSmlVcA84LDSvoiInte3lDnlOeA1wGuBvXu284H5Pd/fVlXVQ6Q0V4KDFyVJWl813Q3nARdH\nxDzgdvLTDuOBiwEi4hxgakrplJ5Bjff1PjkilgArU0r3D6bwodB7roRddy22FkmSak3FISGldGXP\nnAhnA5OAO4GZKaWlPYdMBtqGrsThs/32+auDFyVJWl9VAxdTSnOAOf387NSNnPsFauRRyE03hcmT\n7W6QJKmcpl27ocS5EiRJKq/pQ4JzJUiSVF7Th4S2NrsbJEkqp+lDQqkloYLpIiRJagqGhPY8mdIz\nzxRdiSRJtaXpQ0LvuRIkSdJaTR8SSrMuOnhRkqS+mj4kbLstbLKJLQmSJK2r6UPCqFF55kVbEiRJ\n6qvpQwI4V4IkSeUYEnCuBEmSyjEkYEuCJEnlGBLIIeHxx+Hll4uuRJKk2mFIIHc3rF4NTzxRdCWS\nJNUOQwLOlSBJUjmGBJx1UZKkcgwJwJZbQmurLQmSJPVmSOjhEw6SJPVlSOjhXAmSJPVlSOhhS4Ik\nSX0ZEnq0t9uSIElSb4aEHm1t8PTT8OKLRVciSVJtMCT0KM2VYGuCJEmZIaGHIUGSpL4MCT222w4i\nHLwoSVKJIaHHmDEwZYotCZIklRgSemlrsyVBkqQSQ0IvzpUgSdJahoRenCtBkqS1DAm9lLobUiq6\nEkmSimdI6KW9HVauhKeeKroSSZKKZ0joxbkSJElay5DQS1tb/vrQQ8XWIUlSLTAk9DJxInR0wMc+\nBosWFV2NJEnFMiT0EgFXXw2jRsFRR8GyZUVXJElScQwJ65g6FX7xizwu4bjj4G9/K7oiSZKKYUgo\n41Wvyi0Kv/0tnHoqrFlTdEWSJI08Q0I/DjwQLrsMurrgU58quhpJkkbe6KILqGXveAc89hjMmpWf\nfPjQh4quSJKkkWNI2IiPfCTPwvjhD+flpI87ruiKJEkaGXY3DMBXvpJbFd71LrjllqKrkSRpZFQV\nEiLijIhYEBErIuLWiJi+gWMPiIjfRMRTEbE8Iu6PiI9UX/LIGzUKLrkE9t0XjjkGHnig6IokSRp+\nFYeEiDgR+CowG9gHuAuYGxET+jnlReCbwIHA7sAXgf8bEf9YVcUFGTsWfvITmDwZ3vxmePLJoiuS\nJGl4VdOSMAu4IKV0aUppPnA6sBx4b7mDU0p3ppR+mFK6P6X0aErpCmAuOTTUla22ynMovPQSHH00\nvPBC0RVJkjR8KgoJETEG6ABuLO1LKSXgBmD/Ab7HPj3H/qqSz64V7e05KPzpT3mcwqpVRVckSdLw\nqLQlYQLQAixeZ/9iYPKGToyIhRGxErgd+HZK6aIKP7tm7L03/PjHcMMN8P73Q0pFVyRJ0tAbyUcg\n3whsDrwe+FJEPJRS+uGGTpg1axatra199nV2dtLZ2Tl8VQ7Q4YfD978PJ5+cWxfOOqvoiiRJzaar\nq4uurq4++5YN4cJDkSr4Z3BPd8Ny4O0ppat77b8YaE0pDWgWgYj4DPDulNIe/fx8GjBv3rx5TJs2\nbcD1FeGcc+DTn4bvfQ/+sa6GYkqSGlF3dzcdHR0AHSml7sG8V0XdDSmlVcA84LDSvoiInteVzCDQ\nAmxayWfXqn/5F/jAB+D00+Gaa4quRpKkoVNNd8N5wMURMY88vmAWMB64GCAizgGmppRO6Xn9QeBR\nYH7P+QcDHwO+PqjKa0QEfPObefrmd7wDrr8e9t8/75ckqZ5VHBJSSlf2zIlwNjAJuBOYmVJa2nPI\nZKCt1ymjgHOAHYCXgYeBf04pfXcQddeUlpa8ENRhh8EBB+Q5Fdrb8/aKV+St9/fbbQebbFJ01ZIk\nbVhVAxdTSnOAOf387NR1Xn8L+FY1n1NPxo/PTztcfz385S95e/RRuOuuvOz00qVrj42AqVPXDw/t\n7TBlCkyaBBMnGiQkScVygachtNlm8La3lf/ZihU5NJTCQ+8gceutsGgRvPxy33O22ioHhm237fu1\n3L7NNx/+P58kqbkYEkbIuHGw2255K2f1anjiiTzd85IlsHhx3krfL1kC8+fnr0uXwpo1fc8fPx52\n2inPBHnMMfD61+duEEmSqmVIqBEtLbD99nnbmNWr4Zln1g8Rd96Z52740pdgwgQ46qgcGGbOhC22\nGP4/gySpsRgS6lBLSx6zMHHi+j9bvRpuvz2Pg/jZz+DSS/PYhkMOyYHhmGPy+AdJkjamqqWiVbta\nWvIjmOecA/fcAw8/DOeem7snZs2CHXbI00p/9rNw223rd1tIklRiSGhwO+0EH/5wfuriqafghz+E\nvfaC73wnj1uYOhXe9z746U9drEqS1JchoYm0tsIJJ8APfpDHMdx8M/zDP8Bvf5ufynjNa+Cqq1yw\nSpKUGRKa1OjRcOCBuSti/nzo7oYdd4S//3s46KD8WKYkqbkZEgTAPvvAtdfC3Lnw3HN5XMMJJ+Qx\nDZKk5mRIUB8zZuRWhYsugltugT32yAMen3666MokSSPNkKD1tLTAe94DDz4IZ50FF14IO++cuyZW\nriy6OknSSDEkqF/jx8OnPw0PPQQnnQSf+hTsvjtcfrmPTkpSMzAkaKO23Ra+/W249948duHd74Z9\n94Wbbiq6MknScDIkaMB22y0/InnzzblL4tBD8wyO991XdGWSpOFgSFDFDjwwPyL5n/+ZWxf23DOP\nYbjppjwttCSpMRgSVJUIOPFEuP9++MpX4Ne/zi0L7e3w0Y/CHXc4KZMk1TtDggZl003zI5J//jP8\n7nfw9rfngY3Tp8Ouu8Ls2XmyJklS/TEkaEhE5LUgvvENeOwxuO663C3x9a/nuRamTcuPUC5cWHSl\nkqSBMiRoyI0eDUccAd//fl4j4sc/zvMsfP7zuTvioIPg/PPzglOSpNplSNCwGjsWjjsOfvSjHBgu\nuQQ22ww+9CGYMgWOPjovOHXvvXk6aElS7RhddAFqHltuCSefnLelS3NwuOKK/Lpkiy1g++2hra3v\n197ft7YW92eQpGZiSFAhJk6ED34wb08+mReSWrgQFi3K28KFcM898Itf5J/3flKiFCRKweHgg/sG\nDUnS0DAkqHCTJ+etP6tWwRNPrA0Rvb/eeWce+/Dss3DmmSNXsyQ1A0OCat6YMXnAY3t7+Z9/4hPw\nkY/ANtvkKaMlSUPDkKC696Uv5SclTj0Vtt4ajjqq6IokqTH4dIPqXgR897s5HBx/PNxyS9EVSVJj\nMCSoIYwendeSmD49P1Z5zz1FVyRJ9c+QoIYxbhxcfTW84hUwcyY88kjRFUlSfTMkqKG0tsK11+bA\ncMQRsGRJ0RVJUv0yJKjhTJ6c14544QU48khncpSkahkS1JB22im3KDz8MLztbbByZdEVSVL9MSSo\nYe29N/zsZ3kJ65NOgtWri65IkuqLIUEN7cAD4cor4ac/hdNP7zu9syRpwwwJanjHHAMXXgj/8R/w\nmc8UXY0k1Q9nXFRTOOWUPCvjxz+eF5eaNavoiiSp9hkS1DQ+9rG8RPVHPwoTJsA//EPRFUlSbTMk\nqKmcc04OCqV1Ho4+uuiKJKl2OSZBTSUCLrggj1N4xzvgt78tuiJJql2GBDWd0aOhqwv23Rfe8hb4\n0Y98PFKSyjEkqCmNHZsfi9xvPzjhBNh1V/jWt+DFF4uuTJJqR1UhISLOiIgFEbEiIm6NiOkbOPa4\niLguIpZExLKIuCUiZlRfsjQ0Sus8/P73uVXhzDOhvR0+9zlYvLjo6iSpeBWHhIg4EfgqMBvYB7gL\nmBsRE/o55SDgOuBIYBpwE/CziNi7qoqlIfa61+Xuh4cfzk88fO1reSXJ978fHnig6OokqTjVtCTM\nAi5IKV2aUpoPnA4sB95b7uCU0qyU0ldSSvNSSg+nlD4D/Ak4puqqpWGwww7w9a/Do4/C7Nl52ek9\n9shrP/zmN87WKKn5VBQSImIM0AHcWNqXUkrADcD+A3yPALYAnqnks6WRsvXW8KlPwSOP5FkaH3ww\nT+/8hjfAf/+3gxwlNY9KWxImAC3Auj22i4HJA3yPfwY2A66s8LOlEbXppvDe98I998DPf55fH388\n7LYbzJkDy5cXXaEkDa8RfbohIt4FfA54R0rpqZH8bKlao0blSZd+9Su4/Xbo6IB/+qc8yHH2bAc5\nSmpckSroaO3pblgOvD2ldHWv/RcDrSml4zZw7juB/wCOTyldu5HPmQbMO+igg2htbe3zs87OTjo7\nOwdcszQcFizIAxwvvDB3P5x0Ul4P4jWvKboySc2kq6uLrq6uPvuWLVvGzTffDNCRUuoezPtXFBIA\nIuJW4LaU0pk9rwN4FPhGSuncfs7pJAeEE1NKPx/AZ0wD5s2bN49p06ZVVJ80kp59Fr77XfjmN+Gx\nx2DGjLw2xIwZeXZHSRpp3d3ddHR0wBCEhGq6G84DTouIkyNid+B8YDxwMUBEnBMRl5QO7uliuAT4\nGPD7iJjUs205mMKlWrDVVvDJT+aWhcsvzytNvvnNsOeeuZVh5cqiK5Sk6lUcElJKVwIfB84G/gDs\nBcxMKS3tOWQy0NbrlNPIgx2/DTzea/t69WVLtWXMGHjXu+COO+DXv4ZddoHTTsvjFr7wBViypOgK\nJalyVQ1cTCnNSSntkFIal1LaP6V0R6+fnZpSOrTX6zellFrKbGXnVZDqWQQcdBD85Cd5IqYTToAv\nfzmHhdNOg3vvLbpCSRo4126QhskrX5nXg1i4EM46C665Jg9sPPJIuO46J2eSVPsMCdIw23pr+Jd/\nyeMWfvCD/MjkzJmw1155kSnDgqRaZUiQRsgmm8C73w3z5sFNN8GUKXnK56OPhj/9qejqJGl9hgRp\nhEXAIYfA3Ll57ML99+duiE9/2qWqJdUWQ4JUkAh461vhvvvyWhHnnQe77w5XXmkXhKTaYEiQCjZu\nXB7YeN99ecrnE0+Eww7zSQhJxTMkSDVip51y98M118CiRfDa18LHPgbPPVd0ZZKalSFBqjFHHgl3\n3w1nnw3nnw+77pqfirALQtJIMyRINWjTTfM4hfnz4eCD4eST4cAD4c47i65MUjMxJEg1rK0NfvhD\nuPHGvJhURweccQY880zRlUlqBoYEqQ4cemhuRfjKV3LXw6675tkcn3++6MokNTJDglQnxoyBWbPy\nmhBHHw1nnpknZDrtNPj97x2zIGnoGRKkOjNlClxyCTzyCPzzP+dJmfbdF/bZB+bMgWXLiq5QUqMw\nJEh1qq0NZs/Oa0Jccw3suCN8+MM5RLznPXDLLbYuSBocQ4JU51pa8mOTV12VV5z87Gfh5pvhgAPy\ndM///u8OdJRUHUOC1ECmTMlrQDz0EFx/Pbz61blLYurUvLjUr39t64KkgTMkSA1o1Cg4/PC8DsSi\nRfDFL8Ltt+eFpXbfHc49F556qugqJdU6Q4LU4LbdNrcmPPBAXqL6da+Dz30Ott8eTjkFbrvN1gVJ\n5RkSpCZRWqL68svXti78z//A618P06fDRRfBihVFVymplhgSpCY0YUJuXfjTn+DnP8+tDe97H2y3\nHXz84/Dww0VXKKkWGBKkJtbSkidmuuaaHBje977covDKV8JRR8H/+3+wenXRVUoqiiFBEgA775wH\nNC5aBBdeCEuWwFvekgPDl7/sQEepGRkSJPUxbhyceirccUce1HjggfD5z68d6Hj77UVXKGmkGBIk\n9WvfffMU0IsWwdln50ma9tsvT9L0oQ/lFSoff7zoKiUNF0OCpI2aMAE+8Yk8SdPPfpaDwnXXwTvf\nmQc77rxzngr6wgvhwQd9pFJqFKOLLkBS/WhpyeMU3vKW/PrJJ+E3v8mPUv7P/+RlrNesgUmT4I1v\nzF0VBx4Ie++dz5VUXwwJkqo2eTIcf3zeIK9A+bvfrQ0Nn/gE/O1vsMUW8IY3rA0N++0Hm25abO2S\nNs6QIGlyU3P6AAALTElEQVTItLbCm9+cN4CVK/MAyFJo+PKX8wJUY8fmlobDDoNDD4Vp02C0fxtJ\nNccxCZKGTSkMfOpTeS6GZ56B7m7413+FTTbJX/fbL495eOtb84qV99zjmAapVpjdJY2YlhbYZ5+8\nffSjsGoV/P738Mtfwo03ru2e2Hbb3MJw6KG5tWHHHfO00pJGliFBUmHGjMljFd7whtwNsWIF/Pa3\na0PDlVfmgZCveMXaronp02HLLfM4h/HjDQ/ScDIkSKoZ48blJa4PPzy/XrYMfv3rtaHh+9/ve3wE\nbL752m2LLdb/vve+1laYMQN22mlk/1zLl8O11+bP32svmDhxZD9fqpYhQVLNam2FY4/NG8DixXDf\nffDCC/D88/lr7+9771u6FBYs6Lv/uefyWhQzZ8IHPpDXrRjOAZP33w8XXJAnpPrrX9funzwZ9twz\nB4a99srf77FHHsMh1RJDgqS6MWlS3qq1fHnuwvjOd+Btb8sTQZ12GvzjP+bvh8JLL8FVV8H55+dW\nkIkT4fTT8+JZq1fD3XfDH/+Yt6uugq9+NZ/X0gK77bZ+eGhvt0tFxYlUg8OII2IaMG/evHlMmzat\n6HIkNaDu7vyv/Msvz49qHnts/mV++OEwqornvhYsgO9+N886uXQpHHxwfr/jjtvwnBDPP5+f6Ogd\nHu6+e23Lw5Zb5rCw666w9dbwd38HW221/tfS97ZGqLu7m46ODoCOlFL3YN7LkCCpqT33HFx2WW5d\nuOeePMX0+9+fp5ne2NiBl1/Oy2mffz7MnZu7R045JZ+/xx7V15RSXi+jFBj++Ef4859zcHj22byt\nWlX+3LFjyweIV75ybevETjtVF4RUHwwJkjTEUsqzRZ5/fu6SSCnPJPmBD8ABB/Rt8n/ssdxi8L3v\n5V/m++2XWw1OOCE/cTESta5YkcNCKTj09/XZZ+Hpp2H+/LXLfW+2Gbz61X27NfbcE7bZZvhr1/Ab\nypDgmARJIoeA0uOY552XBxuefz5ccUX+hXr66bmV4Xvfg6uvzv9iP+mkvH+ffUa+1vHj8zbQsRQp\n5YGfvVsn7rgDLr00z00BMHXq2tBQ+rr77k6h3cxsSZCkfqxZkx+/PP98+MlP8sDDPffMrQsnnZTH\nC9S7l1/OK3eWgkPp61/+kn8+enQOR21tOZBsv/362zbbDG5w5cqVebGwJ55Y/+uqVXmejB12yNuO\nO+bPLGoa7zVrcovM44/n+npvvfctWZKfYtl99/W3iROHdzCq3Q2SNMKeeCL/S3zvvZvjaYNly9YO\nqHzggdzFsmhR3h5/PAemkk03LR8ettsOpkzJj5/2FwKefLLv46GQA8CkSfnc0aPh0UfzZ5a0tOT3\nLwWHdbeNhYg1a/KTLus+Qrvu62efXf+X/+LFOVj1NmFCrnXKlNwaM2VKDgJPPJG7eebPh4cfzp8L\neYxIufCw005DE37sblBZXV1ddHZ2Fl1G3fG6Va4Zr1npl8Bg1NN1a23NYzEOOGD9n61enX9ZlkJD\n7wCxcGEe27Fo0dpujN7vOXlyvo6TJ+fAVfq+99ett147sLJ0zVauzO+9YAE88sja7cEH4brr8i/k\nkpaW3PLR3p67WdYNAS++uPH1QUoDQEv/3V/7WjjyyLWvS4Fg0qS8DsnGvPRSDgql0DB/fg5h//Vf\nuT7IM5Duskse9Lr33vD5z2/8fYdbVS0JEXEG8HFgMnAX8E8ppd/3c+xk4KvA64BdgH9PKX10I+9v\nS0IVjj32WK6++uqiy6g7XrfKec2q00zXLaXcLP/EE3nWy0mTqhvUOdBrtnJlbnHoHSD+8pccGMrN\nyLmh15ttNnLdGSn1bXEobRH5iZlqFNqSEBEnkn/p/x/gdmAWMDcidk0pPVXmlE2BJcAXe46VJDW4\niNzkPlJTUI8dm+eS2HXXkfm8oRKRWySmTs1rk9Saap6UnQVckFK6NKU0HzgdWA68t9zBKaW/pJRm\npZQuA56rvlRJkjSSKgoJETEG6ABuLO1Lub/iBmD/oS1NkiQVqdLuhglAC7B4nf2Lgd2GpKJsLMD9\n998/hG/Z+JYtW0Z396C6n5qS161yXrPqeN0q5zWrXK/fnYOepLuigYsRMQV4DNg/pXRbr/1fAg5K\nKW2wNSEibgL+MICBi+8CLh9wYZIkaV0npZSuGMwbVNqS8BSwGlh3HbZJwJODKWQdc4GTgEeAlUP4\nvpIkNbqxwA7k36WDUlFISCmtioh5wGHA1QARET2vvzHYYnp9ztPAoNKPJElN7JaheJNqngQ9D7i4\nJyyUHoEcD1wMEBHnAFNTSqeUToiIvYEANgcm9rz+W0rJQQeSJNWoikNCSunKiJgAnE3uZrgTmJlS\nWtpzyGSgbZ3T/gCUBj9MA94F/AXYqZqiJUnS8KvJtRskSVLxqplMSZIkNQFDgiRJKqvmQkJEnBER\nCyJiRUTcGhHTi66plkXE7IhYs852X9F11ZKIODAiro6Ix3quz7Fljjk7Ih6PiOURcX1E7FJErbVk\nY9ctIi4qc+9dU1S9tSAiPhURt0fEcxGxOCKuioj1VhPwfltrINfMe219EXF6RNwVEct6tlsi4s3r\nHDPo+6ymQkKvxaNmA/uQV5ic2zNQUv27hzyIdHLP9sZiy6k5m5EH2H6QtQNo/1dEfBL4EHnRsn2B\nF8n33QAWgG1oG7xuPX5B33uvPtZBHj4HAt8E9gMOB8YA10XEuNIB3m/r2eg16+G91tdC4JPkhwE6\ngF8CP42IPWAI77OUUs1swK3kpaRLrwNYBHyi6NpqdSMHqu6i66iXDVgDHLvOvseBWb1ebwmsAE4o\nut5a2fq5bhcBPy66tlreyFPZrwHe2Guf91vl18x7bWDX7mng1J7vh+Q+q5mWBBePGpRX9jQJPxwR\nl0XEuo+gqh8RsSP5XyW977vngNvwvhuIQ3qaiOdHxJyI2LrogmrM35FbYZ4B77cB6nPNevFe60dE\njIqId5LnLLplKO+zmgkJbHjxqMkjX07duBV4DzCTvGz3jsDNEbFZkUXVkcnkv5C87yr3C+Bk4FDg\nE8DBwDU9s7A2vZ7r8HXgNyml0jgh77cN6OeagfdaWRHxmoh4HngJmAMcl1J6gCG8z6qZcVE1JKXU\ne27ueyLidvJEVSeQm+ikYZFSurLXy3sj4m7gYeAQ4KZCiqotc4BXAQcUXUgdKXvNvNf6NR/YG2gF\njgcujYiDhvIDaqklYaQWj2poKaVlwINA046WrtCT5LEv3neDlFJaQP7/uOnvvYj4FnAUcEhK6Yle\nP/J+68cGrtl6vNeylNLLKaU/p5T+kFL6DHmw/5kM4X1WMyEhpbQKKC0eBfRZPGpIFqpoBhGxOfl/\nnA3+T6as5y+bJ+l7321JHmntfVeBiNge2IYmv/d6ftm9FXhTSunR3j/zfitvQ9esn+O918obBWw6\nlPdZrXU3bHDxKK0vIs4FfkbuYtgO+AKwCugqsq5a0jM+YxdysgbYqWeRsWdSSgvJfaCfjYiHyMuT\nf5H8VM1PCyi3ZmzouvVss4H/Jv9ltAvwJXIr1qCXp61XETGH/GjescCLEVH6l9yylFJp2Xvvt142\nds167kPvtXVExL+Rx2o8CmwBnEQeqzGj55Chuc+KfmSjzCMcH+z5A60Afge8ruiaankjh4FFPdfr\nUfIS2zsWXVctbT3/46whd2f13r7f65izyI8MLSf/xbNL0XUXvW3oupHXq7+W/Jf2SuDPwHeAiUXX\nXfA1K3e9VgMnr3Oc99sAr5n3Wr/X7T96rsWKnmtzHXDoOscM+j5zgSdJklRWzYxJkCRJtcWQIEmS\nyjIkSJKksgwJkiSpLEOCJEkqy5AgSZLKMiRIkqSyDAmSJKksQ4IkSSrLkCBJksoyJEiSpLL+P6fT\nFHnQ3rrdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x22a14432898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plotting losses\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##########################################################################################################################\n",
    "#saving the list of words used for indexing/tokenizing\n",
    "#rerun of programs will give different order/sets of  unique words and different weights\n",
    "#while evaluating the saved weights it is necessary to use correspondin set of saved uniq_words as look up\n",
    "new_file = 'C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\uniq_saved_new.txt'\n",
    "with open(new_file, 'w') as f:\n",
    "    for item in uniq_words:\n",
    "        f.write(\"%s\\n\" % item)\n",
    "#####################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "russian\n",
      "nations\n",
      "national\n",
      "premises\n",
      "stage\n",
      "first\n",
      "acknowledgments\n",
      "challenges\n",
      "responsible\n",
      "enemies\n",
      "time\n",
      "islamic\n",
      "postwar\n",
      "successors\n",
      "state\n",
      "digital\n",
      "asian\n",
      "forswearing\n",
      "thirty\n",
      "interpretations\n",
      "do\n",
      "issue\n",
      "cold\n",
      "congress\n",
      "countries\n",
      "fair\n",
      "is\n",
      "made\n",
      "cataclysm\n",
      "most\n",
      "outside\n",
      "order\n",
      "replied\n",
      "ronald\n",
      "tradition\n",
      "pride\n",
      "minimal\n",
      "myself\n",
      "challenge\n",
      "condition\n",
      "an\n",
      "era\n",
      "defense\n",
      "ottoman\n",
      "frequent\n",
      "factor\n",
      "great\n",
      "revolutionary\n",
      "strategies\n",
      "rights\n",
      "western\n",
      "embracing\n",
      "their\n",
      "twenty\n",
      "presidency\n",
      "adopting\n",
      "woodrow\n",
      "democratic\n",
      "saudi\n",
      "pluralistic\n",
      "human\n",
      "disorder\n",
      "this\n",
      "president\n",
      "aimed\n",
      "where\n",
      "version\n",
      "presidents\n",
      "understanding\n",
      "arabia\n",
      "city\n",
      "nuclear\n",
      "definition\n",
      "partnership\n",
      "technology\n",
      "tide\n",
      "some\n",
      "instances\n",
      "introduction\n",
      "wars\n",
      "like\n",
      "cooperative\n",
      "kansas\n",
      "reiler\n",
      "philosophical\n",
      "reality\n",
      "world\n",
      "european\n",
      "iranian\n",
      "concepts\n",
      "speech\n",
      "fact\n",
      "all\n",
      "or\n",
      "called\n",
      "participatory\n",
      "acting\n",
      "age\n",
      "rules\n",
      "between\n",
      "them\n",
      "ambivalent\n",
      "play\n",
      "syrian\n",
      "totally\n",
      "decline\n",
      "narrative\n",
      "mankind\n",
      "wilson\n",
      "approaches\n",
      "different\n",
      "china\n",
      "wanted\n",
      "sovereignty\n",
      "defeated\n",
      "multiplicity\n",
      "then\n",
      "preservation\n",
      "century\n",
      "kissinger\n",
      "breakdown\n",
      "both\n",
      "richard\n",
      "proud\n",
      "renewal\n",
      "only\n",
      "policy\n",
      "today\n",
      "these\n",
      "sick\n",
      "united\n",
      "regional\n",
      "legitimacy\n",
      "course\n",
      "humane\n",
      "remembered\n",
      "reagan\n",
      "important\n",
      "iran\n",
      "systems\n",
      "superpower\n",
      "norms\n",
      "liberal\n",
      "on\n",
      "vision\n",
      "statecraft\n",
      "vienna\n",
      "vehemence\n",
      "here\n",
      "spring\n",
      "think\n",
      "conscious\n",
      "eloquence\n",
      "purpose\n",
      "theodore\n",
      "what\n",
      "foreign\n",
      "so\n",
      "taken\n",
      "henry\n",
      "new\n",
      "war\n",
      "follow\n",
      "common\n",
      "dilemmas\n",
      "followed\n",
      "middle\n",
      "him\n",
      "balance\n",
      "expanding\n",
      "islamism\n",
      "many\n",
      "aftermath\n",
      "observing\n",
      "much\n",
      "contribution\n",
      "perspective\n",
      "concept\n",
      "they\n",
      "his\n",
      "westphalian\n",
      "faces\n",
      "similar\n",
      "roosevelt\n",
      "question\n",
      "metternich\n",
      "nancy\n",
      "proliferation\n",
      "uniqueness\n",
      "india\n",
      "franklin\n",
      "based\n",
      "truman\n",
      "with\n",
      "history\n",
      "ushered\n",
      "lion\n",
      "enhancement\n",
      "two\n",
      "consciousness\n",
      "from\n",
      "westphalia\n",
      "yet\n",
      "attributes\n",
      "i\n",
      "conscience\n",
      "there\n",
      "regions\n",
      "he\n",
      "embrace\n",
      "community\n",
      "victories\n",
      "conclusion\n",
      "by\n",
      "confrontation\n",
      "academic\n",
      "when\n",
      "shared\n",
      "played\n",
      "possible\n",
      "revolution\n",
      "iraq\n",
      "urge\n",
      "toward\n",
      "notes\n",
      "longer\n",
      "enigma\n",
      "cllaracler\n",
      "operation\n",
      "empire\n",
      "europe\n",
      "french\n",
      "american\n",
      "vietnam\n",
      "evolution\n",
      "as\n",
      "years\n",
      "not\n",
      "end\n",
      "no\n",
      "khomeini\n",
      "would\n",
      "conciliations\n",
      "consensus\n",
      "changes\n",
      "share\n",
      "done\n",
      "character\n",
      "japan\n",
      "reflections\n",
      "we\n",
      "nixon\n",
      "values\n",
      "economic\n",
      "cyber\n",
      "inexorably\n",
      "governance\n",
      "go\n",
      "man\n",
      "equilibrium\n",
      "bismarck\n",
      "east\n",
      "governments\n",
      "asia\n",
      "korean\n",
      "has\n",
      "continued\n",
      "arab\n",
      "exhortations\n",
      "reflect\n",
      "varieties\n",
      "be\n",
      "role\n",
      "vast\n",
      "palestinian\n",
      "above\n",
      "reflected\n",
      "conquest\n",
      "future\n",
      "our\n",
      "period\n",
      "parties\n",
      "beginning\n",
      "often\n",
      "contents\n",
      "allies\n",
      "peace\n",
      "delivering\n",
      "took\n",
      "back\n",
      "experience\n",
      "territorial\n",
      "respecting\n",
      "had\n",
      "young\n",
      "other\n",
      "found\n",
      "stakeholders\n",
      "uphold\n",
      "afghanistan\n",
      "harry\n",
      "brought\n",
      "penguin\n",
      "\n"
     ]
    }
   ],
   "source": [
    "############################################################################################################\n",
    "#reading the file\n",
    "read_file = \"C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\word2vec\\\\SGNS\\\\uniq_saved_new.txt\"\n",
    "f = open(read_file, \"r\")\n",
    "print(f.read())\n",
    "#############################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "#Alternative way of saving and reading the file using json\n",
    "import json\n",
    "#a = [1,2,3,4,5]\n",
    "d = uniq_words\n",
    "with open('C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\weights_and_words\\\\uniq_saved10.txt', 'w') as f:\n",
    "    f.write(json.dumps(d))\n",
    "\n",
    "#Now read the file back into a Python list object\n",
    "with open('C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\weights_and_words\\\\uniq_saved10.txt', 'r') as f:\n",
    "    c = json.loads(f.read())\n",
    "####################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['russian', 'nations', 'national', 'premises', 'stage']"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#using new networks\n",
    "#state_dict can be used but weights have to be seperated before transferring\n",
    "\n",
    "embed_size = 30\n",
    "fc1 = nn.Linear(len(uniq_words), embed_size, bias = False)\n",
    "pretrained_weights = torch.load('C:\\\\Users\\\\Mnsh\\\\Documents\\\\data\\\\pytorch\\\\word2vec\\\\SGNS\\\\weights_and_words\\\\fc_midl_wt.pth')\n",
    "fc11.weight = pretrained_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-0.5225, -0.2186, -0.6126,  ..., -0.5997,  0.3447, -1.5748],\n",
       "        [ 1.3043, -0.0870, -0.4282,  ..., -0.0965,  0.0154,  1.0580],\n",
       "        [ 0.3189,  0.0833, -0.0369,  ...,  0.2913,  0.2884,  0.7364],\n",
       "        ...,\n",
       "        [ 0.0804,  1.0413,  1.1097,  ...,  0.4449,  0.7431,  1.1347],\n",
       "        [ 0.1467, -0.3945,  0.1304,  ..., -1.5804, -0.6592, -0.1812],\n",
       "        [ 1.4072,  0.5528,  0.1567,  ...,  0.6486,  1.7909, -0.0341]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrained_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0000, 0.9630, 0.9551, 0.9418, 0.9222])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['henry', 'history', 'kissinger', 'character', 'reflections']"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_dist_new(fc1, uniq_words, 'henry', 5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:Anaconda3]",
   "language": "python",
   "name": "conda-env-Anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}