notes/nlp/nlp_Glove-Matrix-Factorization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package words to /home/yui/nltk_data...\n",
      "[nltk_data]   Package words is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/yui/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/yui/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package brown to /home/yui/nltk_data...\n",
      "[nltk_data]   Package brown is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /home/yui/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words:  100554\n",
      "Number of sentences:  4623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/2000 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique words (including capitalized) 14394\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000/2000 [00:03<00:00, 646.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique words:  3973\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import pairwise_distances\n",
    "import pandas as pd\n",
    "import math\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from nltk import word_tokenize\n",
    "nltk.download('words')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from tqdm import tqdm,tqdm_pandas\n",
    "nltk.download('brown')\n",
    "from nltk.corpus import brown\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "import random\n",
    "\n",
    "words = brown.words(categories=\"news\")\n",
    "sents = brown.sents(categories=[\"news\"])\n",
    "\n",
    "print(\"Number of words: \",len(words))\n",
    "print(\"Number of sentences: \",len(sents))\n",
    "print(\"Number of unique words (including capitalized)\",\n",
    "      len(set(words)))\n",
    "\n",
    "def get_wordnet_pos(word):\n",
    "    \"\"\"Map POS tag to first character lemmatize() accepts\"\"\"\n",
    "    tag = nltk.pos_tag([word])[0][1][0].upper()\n",
    "    tag_dict = {\"J\": wordnet.ADJ,\n",
    "                \"N\": wordnet.NOUN,\n",
    "                \"V\": wordnet.VERB,\n",
    "                \"R\": wordnet.ADV}\n",
    "    return tag_dict.get(tag,wordnet.NOUN)\n",
    "\n",
    "def lemmWord(w):\n",
    "    return lem.lemmatize(w,get_wordnet_pos(w))\n",
    "\n",
    "def processCorpus(sents):\n",
    "    w2id,id2w,idx={},{},2\n",
    "    eng = set(nltk.corpus.words.words())\n",
    "    w2id['<s>'],id2w[0]=0,'<s>'\n",
    "    w2id['<e>'],id2w[1]=1,'<e>'\n",
    "    newSents = []\n",
    "    for sent in tqdm(sents):\n",
    "        newSent = [0]\n",
    "        for word in sent:\n",
    "            word = word.lower()\n",
    "            word = lemmWord(word)\n",
    "            if word not in eng:\n",
    "                continue\n",
    "            if word not in w2id:\n",
    "                w2id[word]=idx\n",
    "                id2w[idx]=word\n",
    "                idx+=1\n",
    "            newSent.append(w2id[word])\n",
    "        newSent.append(1)\n",
    "        newSents.append(newSent)\n",
    "    return w2id,id2w,newSents\n",
    "\n",
    "lem = WordNetLemmatizer()\n",
    "w2id,id2w,newSents=processCorpus(sents[:2000])\n",
    "print(\"Number of unique words: \",len(w2id))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Original Matrix Factorization for User-Movie recommender system\n",
    "The model is given by, \n",
    "\n",
    "$$\\hat{r}_{ij}=w_i^\\top u_j + b_i+c_j+\\mu$$\n",
    "\n",
    "where user bias $b_i$, movie bias $c_j$ and global mean $\\mu$.\n",
    "So the alternating least squares now become,\n",
    "\n",
    "$$\\begin{align*}\n",
    "w_i&=\\bigg(\\sum_{j\\in \\Psi_i}u_ju_j^\\top\\bigg)^{-1}\\sum_{j\\in\\Psi_i}(r_{ij}-b_i-c_j-\\mu)u_j \\\\\n",
    "u_j&= \\bigg(\\sum_{i\\in\\Omega_j }w_iw_i^\\top\\bigg)^{-1}\\sum_{i\\in\\Omega_j}(r_{ij}-b_i-c_j-\\mu)w_i \\\\\n",
    "b_i &= \\frac{1}{|\\Psi_i|}\\sum_{j\\in\\Psi_i}(r_{ij}-w_i^\\top u_j-c_j-\\mu)\\\\\n",
    "c_j &= \\frac{1}{|\\Omega_j|}\\sum_{i\\in\\Omega_j}(r_{ij}-w_i^\\top u_j-b_i-\\mu)\n",
    "\\end{align*}$$\n",
    "\n",
    "Adding regularization terms into loss function, \n",
    "\n",
    "$$J = \\sum_{i,j\\in\\Omega}(r_{ij}-\\hat{r}_{ij})^2+\\lambda \\bigg(\\|W\\|^2_F+\\|U\\|_F^2+\\|b\\|_2^2+\\|c\\|_2^2\\bigg)$$\n",
    "\n",
    "Finally,\n",
    "\n",
    "$$\\begin{align*}\n",
    "w_i&=\\bigg(\\sum_{j\\in \\Psi_i}u_ju_j^\\top + \\lambda I\\bigg)^{-1}\\sum_{j\\in\\Psi_i}(r_{ij}-b_i-c_j-\\mu)u_j \\\\\n",
    "u_j&= \\bigg(\\sum_{i\\in\\Omega_j }w_iw_i^\\top+ \\lambda I\\bigg)^{-1}\\sum_{i\\in\\Omega_j}(r_{ij}-b_i-c_j-\\mu)w_i \\\\\n",
    "b_i &= \\frac{1}{|\\Psi_i|+\\lambda}\\sum_{j\\in\\Psi_i}(r_{ij}-w_i^\\top u_j-c_j-\\mu)\\\\\n",
    "c_j &= \\frac{1}{|\\Omega_j|+\\lambda}\\sum_{i\\in\\Omega_j}(r_{ij}-w_i^\\top u_j-b_i-\\mu)\n",
    "\\end{align*}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weight(vec,xmax=100,alpha=.75):\n",
    "    vec[vec<1]=(vec[vec<1]/xmax)**alpha\n",
    "    vec[vec>=1]=1\n",
    "    return vec\n",
    "def log_add1(vec):\n",
    "    return np.log(vec+1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Matrix Factorization for Term-Term Matrix\n",
    "The target matrix with elements $X_{ij}$ represent how much time word $i$ appears in context of word $j$. It is calculated as follows,\n",
    "$$X_{ij}\\leftarrow X_{ij}+\\frac{1}{j-i}$$\n",
    "Since it can lead to large value, log scale is applied. To avoid NaN value, one is added before taking the log, \n",
    "\n",
    "$$\\log_{+1}X_{ij}=\\log(X_{ij}+1)$$\n",
    "\n",
    "A weight function is introduced to $X_{ij}$, \n",
    "$$f(X)=\\big(\\frac{X}{X_{max}}\\big)^\\alpha$$\n",
    "where $\\alpha=0.75$ and $X_{max}=100$.\n",
    "\n",
    "This model is predicting log of relations, \n",
    "\n",
    "$$\\log_{+1}X_{ij}\\approx w_i^\\top u_j+b_i+c_j+\\mu$$\n",
    "\n",
    "The cost function is given by, \n",
    "\n",
    "$$J=\\sum^N_{i=1}\\sum^M_{j=1}f(X_{ij})\\bigg(w_i^\\top u_j-\\log_{+1}X_{ij}\\bigg)^2$$\n",
    "\n",
    "The alternating least squares become,\n",
    "\n",
    "$$\\begin{align*}\n",
    "w_i &= \\bigg(\\sum_{j=1}^M f(X_{ij})u_ju_j^\\top\\bigg)^{-1}\\sum^M_{j=1}f(X_{ij})\\bigg(\\log_{+1}X_{ij}-b_i-c_j-\\mu\\bigg)u_j \\\\\n",
    "u_j &= \\bigg(\\sum_{i=1}^Nf(X_{ij})w_iw_i^\\top\\bigg)^{-1}\\sum_{i=1}^Nf(X_{ij})\\bigg(\\log_{+1}X_{ij}-b_i-c_j-\\mu\\bigg)w_i\\\\\n",
    "b_i &= \\frac{1}{\\sum^M_{j=1}f(X_{ij})} \\sum^M_{j=1}f(X_{ij})\\bigg(\\log_{+1}X_{ij}-w_i^\\top u_j-c_j-\\mu\\bigg)\\\\\n",
    "c_j &= \\frac{1}{\\sum^N_{i=1}f(X_{ij})} \\sum^N_{i=1}f(X_{ij})\\bigg(\\log_{+1}X_{ij}-w_i^\\top u_j-b_i-\\mu\\bigg)\n",
    "\\end{align*}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MF:\n",
    "    def __init__(self,V,CS=10,k=50):\n",
    "        self.CS = CS\n",
    "        self.V = V\n",
    "        self.k = k\n",
    "        self.X = np.zeros((V,V))\n",
    "        self.W = np.random.randn(V,k)\n",
    "        self.U = np.random.randn(V,k)\n",
    "        self.B = np.random.randn(V)\n",
    "        self.C = np.random.randn(V)\n",
    "        self.reg=1e-2\n",
    "    def setX(self,newSents):\n",
    "        for sent in tqdm(newSents):\n",
    "            n = len(sent)\n",
    "            for i in range(n):\n",
    "                end = i+self.CS+1\n",
    "                if end>n: end=n\n",
    "                for j in range(i+1,end):\n",
    "                    self.X[sent[i],sent[j]]+=1/(j-i)\n",
    "                    self.X[sent[j],sent[i]]+=1/(j-i)\n",
    "        self.fX = weight(self.X)\n",
    "        self.log1X = log_add1(self.X)\n",
    "        self.mu = self.log1X.mean()\n",
    "    def alternatingLS(self,epochs=500):\n",
    "        Js = []\n",
    "        for epoch in tqdm(range(epochs)):\n",
    "            for i in range(self.V):\n",
    "                self.updateW(i)\n",
    "                self.updateB(i)\n",
    "            for j in range(self.V):\n",
    "                self.updateU(j)\n",
    "                self.updateC(j)\n",
    "            loss = self.loss()\n",
    "            Js.append(loss)\n",
    "        return Js\n",
    "    def updateW(self,i):\n",
    "        down=self.reg*np.eye(self.k)+(self.fX[i,:]*self.U.T).dot(self.U)\n",
    "        up=(self.fX[i,:]*(self.log1X[i,:]-self.B[i]-self.C-self.mu)).dot(self.U)\n",
    "        self.W[i]=np.linalg.solve(down,up)\n",
    "    def updateB(self,i):\n",
    "        down=self.fX[i].sum()\n",
    "        up=self.fX[i,:].dot(self.log1X[i,:]-self.W[i].dot(self.U.T)-self.C-self.mu)\n",
    "        self.B[i]=up/down/(1+self.reg)\n",
    "    def updateU(self,j):\n",
    "        down=self.reg*np.eye(self.k)+(self.fX[:,j]*self.W.T).dot(self.W)\n",
    "        up=(self.fX[:,j]*(self.log1X[:,j]-self.B-self.C[j]-self.mu)).dot(self.W)\n",
    "        self.U[j]=np.linalg.solve(down,up)\n",
    "    def updateC(self,j):\n",
    "        down = self.fX[:,j].sum()\n",
    "        up=self.fX[:,j].dot(self.log1X[:,j]-self.W.dot(self.U[j].T)-self.B-self.mu)\n",
    "        self.C[j]=up/down/(1+self.reg)\n",
    "    def loss(self):\n",
    "        tmp = self.W.dot(self.U.T)+self.B.reshape(self.V,1)\n",
    "        tmp+= self.C.reshape(1,self.V)+self.mu-self.log1X\n",
    "        return (self.fX*tmp*tmp).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000/2000 [00:00<00:00, 9952.35it/s]\n"
     ]
    }
   ],
   "source": [
    "mf = MF(len(w2id))\n",
    "mf.setX(newSents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [07:00<00:00,  4.20s/it]\n"
     ]
    }
   ],
   "source": [
    "epochs = 100\n",
    "Js = mf.alternatingLS(epochs=epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'loss')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAeRklEQVR4nO3dfZRcdZ3n8ffn3qrOM3kgDUISSNDoCjqGmEFmcDysKE/rLuiKwo6aw7CbOXvwLHqcnUF3dnGcYdY548PIrsMMAxnDLAuyIJLjckREB/Xs4aF5FIJI5CkhkTTmmZCku/q7f9xfVVdXV3V3Qqor6f68zimq7u/+7r2/2xX60/f3uw+KCMzMzEaSdboBZmZ2+HNYmJnZqBwWZmY2KoeFmZmNymFhZmajKnW6Ae0wf/78WLx4caebYWZ2RHn44YdfjYjuZvMmZFgsXryYnp6eTjfDzOyIIunFVvPcDWVmZqNyWJiZ2agcFmZmNiqHhZmZjcphYWZmo3JYmJnZqBwWZmY2qraFhaSpkh6U9LikpyT9WSpfIukBSc9K+rakrlQ+JU2vT/MX163r86n8GUnntKvNr+3r52s/eIbHNmxv1ybMzI5I7Tyy2Ae8PyLeBSwDzpV0OvBXwNcjYimwDbgs1b8M2BYRbwG+nuoh6WTgYuAU4FzgbyXlbWlw/wDX/Gg9jzsszMyGaFtYRGF3miynVwDvB25L5WuAC9PnC9I0af5ZkpTKb4mIfRHxPLAeOK0dbS7lAqB/wA+EMjOr19YxC0m5pMeALcA9wK+A7RHRn6psBBakzwuADQBp/g7g6PryJsvUb2uVpB5JPb29vQfV3lKWwqIycFDLm5lNVG0Ni4ioRMQyYCHF0cDbm1VL72oxr1V547aui4gVEbGiu7vpfbBGVcqKH4ePLMzMhhqXs6EiYjvwz8DpwBxJ1RsYLgQ2pc8bgUUAaf5sYGt9eZNlDqly6obq85GFmdkQ7TwbqlvSnPR5GvAB4Gngx8BHU7WVwJ3p89o0TZr/o4iIVH5xOltqCbAUeLBNbSbPRMVHFmZmQ7TzFuXHAWvSmUsZcGtEfE/SOuAWSX8BPArckOrfAPyTpPUURxQXA0TEU5JuBdYB/cDlEVFpV6PzTPRVHBZmZvXaFhYR8QRwapPy52hyNlNE7AUuarGuq4GrD3Ubmyln8gC3mVkDX8HdoJRnHuA2M2vgsGhQykT/gI8szMzqOSwalHLR7zELM7MhHBYNSlnmAW4zswYOiwbl3N1QZmaNHBYN8kwe4DYza+CwaFDOM586a2bWwGHRwAPcZmbDOSwalLKMPndDmZkN4bBoUMpExQPcZmZDOCwalHLfG8rMrJHDooEHuM3MhnNYNCj51Fkzs2EcFg3yLPPZUGZmDRwWDXwFt5nZcA6LBqXcRxZmZo0cFg3KmejzkYWZ2RAOiwZ5Jio+sjAzG8Jh0aCU+wpuM7NGDosG5dzP4DYza+SwaFDyqbNmZsM4LBqUcl+UZ2bWyGHRoLiC291QZmb1HBYNSnnxDO4IH12YmVW1LSwkLZL0Y0lPS3pK0hWp/IuSXpb0WHqdX7fM5yWtl/SMpHPqys9NZeslXdmuNkNxnQVAxV1RZmY1pTauux/4XEQ8ImkW8LCke9K8r0fEV+orSzoZuBg4BTge+KGkt6bZ3wQ+CGwEHpK0NiLWtaPReV6ERf9AUMrbsQUzsyNP28IiIjYDm9PnXZKeBhaMsMgFwC0RsQ94XtJ64LQ0b31EPAcg6ZZUty1hUc6Kgy0PcpuZDRqXMQtJi4FTgQdS0aclPSFptaS5qWwBsKFusY2prFV54zZWSeqR1NPb23vQbS1Vjyx8rYWZWU3bw0LSTOB24DMRsRO4FngzsIziyOOr1apNFo8RyocWRFwXESsiYkV3d/dBt7eUxiz8tDwzs0HtHLNAUpkiKG6KiO8ARMQrdfP/AfhemtwILKpbfCGwKX1uVX7IlfIiPz3AbWY2qJ1nQwm4AXg6Ir5WV35cXbUPA0+mz2uBiyVNkbQEWAo8CDwELJW0RFIXxSD42na1e/DIwt1QZmZV7TyyOAP4JPBzSY+lsi8Al0haRtGV9ALwhwAR8ZSkWykGrvuByyOiAiDp08DdQA6sjoin2tXocu4BbjOzRu08G+pnNB9vuGuEZa4Grm5SftdIyx1KeeYBbjOzRr6Cu0G57joLMzMrOCwalKrXWfhsKDOzGodFg+p1Fn60qpnZIIdFAx9ZmJkN57BoULuC20cWZmY1DosGtQFuH1mYmdU4LBrUuqF8ZGFmVuOwaJD73lBmZsM4LBqUfW8oM7NhHBYNaqfO+gpuM7Mah0WDsk+dNTMbxmHRIPeps2ZmwzgsGpQz3xvKzKyRw6JB9eFH7oYyMxvksGjgAW4zs+EcFg1K7oYyMxvGYdGgegW3r7MwMxvksGjgZ3CbmQ3nsGiQZSKTB7jNzOo5LJoo5ZkffmRmVsdh0UQ5ExUfWZiZ1Tgsmijlmc+GMjOr47BoopTJA9xmZnUcFk2UcnmA28ysTtvCQtIiST+W9LSkpyRdkcrnSbpH0rPpfW4ql6RrJK2X9ISk5XXrWpnqPytpZbvaXFXK3A1lZlavnUcW/cDnIuLtwOnA5ZJOBq4E7o2IpcC9aRrgPGBpeq0CroUiXICrgPcApwFXVQOmXcq5fNdZM7M6bQuLiNgcEY+kz7uAp4EFwAXAmlRtDXBh+nwBcGMU7gfmSDoOOAe4JyK2RsQ24B7g3Ha1G4pHq7obysxs0LiMWUhaDJwKPAAcGxGboQgU4JhUbQGwoW6xjamsVXnjNlZJ6pHU09vb+4baW84zD3CbmdVpe1hImgncDnwmInaOVLVJWYxQPrQg4rqIWBERK7q7uw+usUkpl+8NZWZWp61hIalMERQ3RcR3UvErqXuJ9L4llW8EFtUtvhDYNEJ525SyjD6HhZlZTTvPhhJwA/B0RHytbtZaoHpG00rgzrryT6Wzok4HdqRuqruBsyXNTQPbZ6eytillot/dUGZmNaU2rvsM4JPAzyU9lsq+AHwZuFXSZcBLwEVp3l3A+cB6YA9wKUBEbJX058BDqd6XImJrG9vt6yzMzBq0LSwi4mc0H28AOKtJ/QAub7Gu1cDqQ9e6kZXzjNf6+8drc2Zmhz1fwd1EKZMvyjMzq+OwaCLPMvrcDWVmVuOwaKKce4DbzKyew6KJUp75OgszszoOiybKmfykPDOzOg6LJnxvKDOzoRwWTZRyD3CbmdVzWDRRzkXF3VBmZjUOiyZKWeZuKDOzOg6LJkq5B7jNzOo5LJooeYDbzGwIh0UTpbx4BndxuyozM3NYNFHKivsf+sI8M7OCw6KJUl6EhW8maGZWcFg0Uc6KH4ufw21mVnBYNFE9snA3lJlZwWHRRHXMwldxm5kVHBZNlPLix9Lvay3MzACHRVPVIwtfa2FmVhhTWEi6QtJRKtwg6RFJZ7e7cZ1Srh1ZOCzMzGDsRxZ/EBE7gbOBbuBS4Mtta1WH5bUjC3dDmZnB2MNC6f184B8j4vG6sgmnnHuA28ys3ljD4mFJP6AIi7slzQIm7J/dpcwD3GZm9UpjrHcZsAx4LiL2SJpH0RU1IfkKbjOzocZ6ZPE7wDMRsV3SJ4A/BXaMtICk1ZK2SHqyruyLkl6W9Fh6nV837/OS1kt6RtI5deXnprL1kq48sN07OLUjC3dDmZkBYw+La4E9kt4F/DHwInDjKMt8Czi3SfnXI2JZet0FIOlk4GLglLTM30rKJeXAN4HzgJOBS1LdtqodWXiA28wMGHtY9Edxv+4LgG9ExDeAWSMtEBE/AbaOcf0XALdExL6IeB5YD5yWXusj4rmI2A/ckuq2VW2A291QZmbA2MNil6TPA58E/m/6i798kNv8tKQnUjfV3FS2ANhQV2djKmtVPoykVZJ6JPX09vYeZNMK1W4oP4fbzKww1rD4OLCP4nqLX1P8wv7rg9jetcCbKQbLNwNfTeXNTsONEcqHF0ZcFxErImJFd3f3QTRtUO57Q5mZDTGmsEgBcRMwW9KHgL0RMdqYRbP1vBIRlYgYAP6BopsJiiOGRXVVFwKbRihvq9oV3A4LMzNg7Lf7+BjwIHAR8DHgAUkfPdCNSTqubvLDQPVMqbXAxZKmSFoCLE3bewhYKmmJpC6KQfC1B7rdAzV46qy7oczMYOzXWfwX4LcjYguApG7gh8BtrRaQdDNwJjBf0kbgKuBMScsoupJeAP4QICKeknQrsA7oBy6PiEpaz6eBu4EcWB0RTx3gPh6wsk+dNTMbYqxhkVWDIvkNoxyVRMQlTYpvGKH+1cDVTcrvAu4aYzsPidxHFmZmQ4w1LL4v6W7g5jT9ccb5F/h4KnuA28xsiDGFRUT8Z0n/FjiD4gyl6yLijra2rINqDz/yRXlmZsDYjyyIiNuB29vYlsNG7RblvijPzAwYJSwk7aL5dQ0CIiKOakurOqzsGwmamQ0xYlhExIi39JioBm8k6G4oMzPwM7ibKrkbysxsCIdFE1kmMvk6CzOzKodFC6U8o8/XWZiZAQ6LlsqZfGRhZpY4LFoo5RkVj1mYmQEOi5ZKmejz2VBmZoDDoqVS7m4oM7Mqh0ULpcwD3GZmVQ6LFsq5PGZhZpY4LFrIfTaUmVmNw6KFcp55gNvMLHFYtFDK5dt9mJklDosWSlnmsDAzSxwWLZQy+a6zZmaJw6IFX2dhZjbIYdFC2TcSNDOrcVi0UMp8nYWZWZXDooU8y+hzN5SZGeCwaKmce4DbzKyqbWEhabWkLZKerCubJ+keSc+m97mpXJKukbRe0hOSltctszLVf1bSyna1t1Ep96mzZmZV7Tyy+BZwbkPZlcC9EbEUuDdNA5wHLE2vVcC1UIQLcBXwHuA04KpqwLRbKRP9HuA2MwPaGBYR8RNga0PxBcCa9HkNcGFd+Y1RuB+YI+k44BzgnojYGhHbgHsYHkBtUfK9oczMasZ7zOLYiNgMkN6PSeULgA119Tamslblw0haJalHUk9vb+8bbmgp9wC3mVnV4TLArSZlMUL58MKI6yJiRUSs6O7ufsMNKufuhjIzqxrvsHgldS+R3rek8o3Aorp6C4FNI5S3XZ6Jio8szMyA8Q+LtUD1jKaVwJ115Z9KZ0WdDuxI3VR3A2dLmpsGts9OZW3nK7jNzAaV2rViSTcDZwLzJW2kOKvpy8Ctki4DXgIuStXvAs4H1gN7gEsBImKrpD8HHkr1vhQRjYPmbeEBbjOzQW0Li4i4pMWss5rUDeDyFutZDaw+hE0bk+p1FhGB1GzoxMxs8jhcBrgPO6WsCAjfH8rMzGHRUikvwsJXcZuZOSxaKmfFj8bP4TYzc1i0VDuy8CC3mZnDopXqmIW7oczMHBYtlfLiR+OruM3MHBYt1Y4s3A1lZuawaKWce4DbzKzKYdFC7usszMxqHBYtlNPZUL5NuZmZw6KlUuYBbjOzKodFCyUfWZiZ1TgsWqgeWXjMwszMYdHS4BXc7oYyM3NYtFAb4PaRhZmZw6KV2gC3jyzMzBwWreS+N5SZWY3DooXqFdy+3YeZmcOipcGHH7kbyszMYdFC9UaCvs7CzMxh0VL1FuUVH1mYmTksWin7yMLMrMZh0ULt4Uc+ddbMzGHRik+dNTMb1JGwkPSCpJ9LekxSTyqbJ+keSc+m97mpXJKukbRe0hOSlo9HG8u5w8LMrKqTRxb/MiKWRcSKNH0lcG9ELAXuTdMA5wFL02sVcO14NM5XcJuZDTqcuqEuANakz2uAC+vKb4zC/cAcSce1uzHlXMycUmLzjr3t3pSZ2WGvU2ERwA8kPSxpVSo7NiI2A6T3Y1L5AmBD3bIbU9kQklZJ6pHU09vb+4YbKIlli+bw6Evb3/C6zMyOdJ0KizMiYjlFF9Plkt43Ql01KRs2kBAR10XEiohY0d3dfUgaufyEOfzi1zt5bV//IVmfmdmRqiNhERGb0vsW4A7gNOCVavdSet+Sqm8EFtUtvhDYNB7tPPXEuQwEPL7BRxdmNrmNe1hImiFpVvUzcDbwJLAWWJmqrQTuTJ/XAp9KZ0WdDuyodle12/JFcwF45KVt47E5M7PDVqkD2zwWuENSdfv/OyK+L+kh4FZJlwEvARel+ncB5wPrgT3ApePV0NnTy7y5ewaPeNzCzCa5cQ+LiHgOeFeT8t8AZzUpD+DycWhaU+8+cS73rHuFiCAFnJnZpHM4nTp7WFp+wly27enj+Vdf63RTzMw6xmExiuUnVsct3BVlZpOXw2IUb+meyaypJQ9ym9mk5rAYRZYVF+c98qLDwswmL4fFGCw/YS6/fGUXu31xnplNUg6LMVjui/PMbJJzWIzBskVzAPh/v3q1wy0xM+sMh8UYzJ5W5gNvP5brf/o867fs6nRzzMzGncNijP7yI+9gxpQSn/n2Y+zv9zMuzGxycViM0TGzpvLfP/JOnnx5J3/zw192ujlmZuPKYXEAzjnlTXxsxUL+7r5fcf9zv+l0c8zMxo3D4gD9t399CifMm87K1Q/y7Ydeorh1lZnZxOawOEAzp5S47T/+LisWz+VPbv85f/R/nmDPfl9/YWYTm8PiIMyfOYUb/+A9XHHWUr7z6EY+8NX7+Kf7X2Rff6XTTTMzawuHxUHKM/HZD76Vm//D6bxp9lT+63ef5My//meu/+lzbNm1t9PNMzM7pDQR+9xXrFgRPT0947a9iOBn61/lmnuf5aEXtpEJ3ru0mw/91nGc+bZujpk1ddzaYmZ2sCQ9HBErms5zWBxa67fs4ruPbuKOR1/m5e2vA3DycUfxe2+dz2+fOI93nziXuTO6OtI2M7OROCw6ICJYt3kn9/2yl/ue6eWRl7bRVyl+1ifNn8E7FszmHQuO4pTjZ7P02Jl0z5ziJ/GZWUc5LA4De/sqPL5hOz0vbuPRl7azbtMONu0YHNuYM73M0mNmsmT+DJbMn8mS+dNZNK94HTW13MGWm9lkMVJYjPszuCerqeWc95x0NO856eha2dbX9rNu006e3bKLX76ym/VbdvGjX/Ty6u6NQ5adPa3M8XOmcfzsqRw/Zxpvmj2VY2ZN4U2zp9I9awrdM6cwd3oXWeYjEzNrD4dFB82b0cV7l87nvUvnDynftbePF17dw4Zte9iwtXjfvH0vL29/nZ4Xt7Hj9b5h68ozMXd6F/NndjFvRhdzZ3Qxb3oXc6eXmT29iznTysyZXmb2tDJHTStz1NQys6aWmN6Vu/vLzEblsDgMzZpa5p0LZ/POhbObzn99f4Utu/by6x17eXX3fnp37aV39z62vrafV3fvZ+tr+3l68062vbaf7a/3MVJPY56JmVNKzJxSYtbUEjOmFK+ZU3KmdxXl07tyZkwpMa2cM70rZ1pXzrTy4PvU9JrWlTO1lNWmcx/pmE0YDosj0LSunBOPnsGJR88YtW5lINi1t48dr/exfU8fO9Pnna/3s3NvH7v39rNrbx+79vaze18/r+3vZ8ee/WzaXmHPvqJsz/4K/QMHPrZVysTUcs6UUla8yjldecaUckZXntFVSq986Hu5/j0X5TyjlGeU0+diWpRzUcqy2nupWjcTpbqywXeRZ8V08S7yVJ5pcL6PtMyGc1hMcHkm5kzvYs70Lk48evT6rezvH2DP/n5e76uwZ3+F1/dX2NtXqU3v7au+Bgbf+yvs7x+c3l8ZYH9/hX39A+xPr937+ovPlWK6r/YetfLxlglKWUaWpXcVP8c8y8gzyCWyFCy1z7UyhpepqJtlKtalIpDyjFo45SrmZRqsl9XXS5+zar1MKK2rWtZsfqah68rqymiYloSotmFwmVqZQHXbEZBlIFRXXkwX66euXipjsF2pCbXli+0Nvhfzh37OVL+9oW2u1qO+zpB5Qlndehu2r4blhtTxHxBHTlhIOhf4BpAD10fElzvcpEmlOAroYs44bzciqAwEfZWgb2CAvhQkfZUB+geC/kox3T+Q3lN5X2WAykCkOsX8SvpciagtWxmIWr1K/Sttt78SDMRgnYE0b6CuTnV+ZYDa58GyYtv7+oNKDO5PZSCISPWj7vPA0M8DQ8qLeZW694GBICjqTMATGw87TYOkSKfm4Qd185oHEUMCanD5tGgtqEYKNOrqn3z8bP7HJace8n0/IsJCUg58E/ggsBF4SNLaiFjX2ZZZu0mpSymHaeSdbs5hLWJ4AA2fTsHDYMDUh1OxnmogFWEVMVh/YACCwe1U34tFq9MMW7Z+nVHX1gga2pKWq9tGtU3RsP4h66lfR1qgut3BddbXH6w7ZDqGl9d6YOvX2WR5hm1jaIjHCMtD/c+jfp9brH/I/FQ//eeEedMO9T8t4AgJC+A0YH1EPAcg6RbgAsBhYZZU/3LN0BHzP7YdOY6UGwkuADbUTW9MZWZmNg6OlLBoNro0pIdW0ipJPZJ6ent7x6lZZmaTw5ESFhuBRXXTC4FN9RUi4rqIWBERK7q7u8e1cWZmE92REhYPAUslLZHUBVwMrO1wm8zMJo0jYhwsIvolfRq4m+LU2dUR8VSHm2VmNmkcEWEBEBF3AXd1uh1mZpPRkdINZWZmHeSwMDOzUU3Ihx9J6gVefAOrmA+8eoiac6SYjPsMk3O/J+M+w+Tc7wPd5xMjounppBMyLN4oST2tnhY1UU3GfYbJud+TcZ9hcu73odxnd0OZmdmoHBZmZjYqh0Vz13W6AR0wGfcZJud+T8Z9hsm534dsnz1mYWZmo/KRhZmZjcphYWZmo3JY1JF0rqRnJK2XdGWn29MukhZJ+rGkpyU9JemKVD5P0j2Snk3vczvd1kNNUi7pUUnfS9NLJD2Q9vnb6UaVE4qkOZJuk/SL9J3/zkT/riV9Nv3bflLSzZKmTsTvWtJqSVskPVlX1vS7VeGa9PvtCUnLD2RbDouk7tGt5wEnA5dIOrmzrWqbfuBzEfF24HTg8rSvVwL3RsRS4N40PdFcATxdN/1XwNfTPm8DLutIq9rrG8D3I+JfAO+i2P8J+11LWgD8J2BFRLyD4uajFzMxv+tvAec2lLX6bs8DlqbXKuDaA9mQw2JQ7dGtEbEfqD66dcKJiM0R8Uj6vIvil8cCiv1dk6qtAS7sTAvbQ9JC4F8B16dpAe8HbktVJuI+HwW8D7gBICL2R8R2Jvh3TXGT1GmSSsB0YDMT8LuOiJ8AWxuKW323FwA3RuF+YI6k48a6LYfFoEn56FZJi4FTgQeAYyNiMxSBAhzTuZa1xd8AfwwMpOmjge0R0Z+mJ+J3fhLQC/xj6n67XtIMJvB3HREvA18BXqIIiR3Aw0z877qq1Xf7hn7HOSwGjfro1olG0kzgduAzEbGz0+1pJ0kfArZExMP1xU2qTrTvvAQsB66NiFOB15hAXU7NpD76C4AlwPHADIoumEYT7bsezRv69+6wGDTqo1snEklliqC4KSK+k4pfqR6WpvctnWpfG5wB/BtJL1B0Mb6f4khjTuqqgIn5nW8ENkbEA2n6NorwmMjf9QeA5yOiNyL6gO8Av8vE/66rWn23b+h3nMNi0KR5dGvqq78BeDoivlY3ay2wMn1eCdw53m1rl4j4fEQsjIjFFN/tjyLi94EfAx9N1SbUPgNExK+BDZLelorOAtYxgb9riu6n0yVNT//Wq/s8ob/rOq2+27XAp9JZUacDO6rdVWPhK7jrSDqf4q/N6qNbr+5wk9pC0nuBnwI/Z7D//gsU4xa3AidQ/A93UUQ0Dp4d8SSdCfxRRHxI0kkURxrzgEeBT0TEvk6271CTtIxiUL8LeA64lOIPxQn7XUv6M+DjFGf+PQr8e4r++Qn1XUu6GTiT4lbkrwBXAd+lyXebgvN/Upw9tQe4NCJ6xrwth4WZmY3G3VBmZjYqh4WZmY3KYWFmZqNyWJiZ2agcFmZmNiqHhdlhQtKZ1bvhmh1uHBZmZjYqh4XZAZL0CUkPSnpM0t+nZ2TslvRVSY9IuldSd6q7TNL96fkBd9Q9W+Atkn4o6fG0zJvT6mfWPXvipnQhFZK+LGldWs9XOrTrNok5LMwOgKS3U1wZfEZELAMqwO9T3KzukYhYDtxHcSUtwI3An0TEb1FcMV8tvwn4ZkS8i+K+RdXbLpwKfIbimSonAWdImgd8GDglrecv2ruXZsM5LMwOzFnAu4GHJD2Wpk+iuG3Kt1Od/wW8V9JsYE5E3JfK1wDvkzQLWBARdwBExN6I2JPqPBgRGyNiAHgMWAzsBPYC10v6CMWtGszGlcPC7MAIWBMRy9LrbRHxxSb1RrqPTrNbRVfV36uoApTSMxhOo7hL8IXA9w+wzWZvmMPC7MDcC3xU0jFQe97xiRT/L1XvaPrvgJ9FxA5gm6TfS+WfBO5Lzw7ZKOnCtI4pkqa32mB67sjsiLiLootqWTt2zGwkpdGrmFlVRKyT9KfADyRlQB9wOcVDhU6R9DDFk9k+nhZZCfxdCoPqHV+hCI6/l/SltI6LRtjsLOBOSVMpjko+e4h3y2xUvuus2SEgaXdEzOx0O8zaxd1QZmY2Kh9ZmJnZqHxkYWZmo3JYmJnZqBwWZmY2KoeFmZmNymFhZmaj+v9O46v2GFVlAwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(len(Js)),Js)\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "Embeddings = (mf.W+mf.U)/2\n",
    "tsne = TSNE()\n",
    "Z = tsne.fit_transform(Embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAD4CAYAAADcpoD8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeViV1drA4d8SQXEINNBETdRUlHlQUZwnKs3ZzMhwzgbNTplmX+eoTZad8mhpToEZiYmGZlo54ISagCAOiSZgph4lEWRUhvX9AewDiEoyy3NfFxd7v9Na7y559rumR2mtEUIIIcpLjYqugBBCiOpFAo8QQohyJYFHCCFEuZLAI4QQolxJ4BFCCFGualZ0BfKzsLDQ1tbWFV0NIYSoUsLCwv7SWltWdD2Kq1IFHmtra0JDQyu6GkIIUaUopc5XdB3+DmlqE0IIUa4k8AghhChXpRJ4lFLmSqkApdRppdRvSqkuSqmGSqkdSqmzub8blEZZQgghqrbSeuL5D/CT1toGcAR+A2YDu7TWbYBdue+FuKfY2Fjat2/P5MmTsbW1ZcCAAaSlpREREYG7uzsODg4MGzaM69evV3RVhRD3ocSBRyn1ENADWA2gtb6ltU4AhgBrcg9bAwwtaVmi+jh79iwvv/wyJ0+exNzcnI0bN/L888/z0UcfERkZib29PfPmzavoagoh7kNpjGprBcQBPkopRyAMeBVorLW+DKC1vqyUalQKZYkHVGD4RRb+HMWlhDQa6kQaWTXHyckJAFdXV86dO0dCQgI9e/YEwNvbm1GjRlVklYUQ96k0mtpqAi7AMq21M5DC32hWU0pNUUqFKqVC4+LiSqE6oqoJDL/IW5uOczEhDQ1cuZHOtXRNYPhFAIyMjEhISKjYSgohSk1pBJ4/gT+11r/mvg8gJxBdUUo1Acj9fbWok7XWK7TWblprN0vLKjP/SZSihT9HkZaRVWCb1pqFP0cZ3puZmdGgQQP2798PwNq1aw1PP0KIqqXEgUdr/V/gglKqXe6mvsApYAvgnbvNG9hc0rLEg+lSQlqxtq9Zs4aZM2fi4OBAREQEzZs35+uvvwbA19eXS5culXldhRAlV1orF0wD/JRSJkA0MJ6coPadUmoi8AcgDfKiSFbmplzMF2RqmjXGauJSrMxNAThw4AAXLlxgw4YNvPrqq0yZMoV69epha2vLmjVrWLp0KTVq1MDOzg4rK6uKug0hRDGVynBqrXVEbnOZg9Z6qNb6utb6mta6r9a6Te7v+NIoSzx4Znq2w9TYqMA2U2MjZnrmPER/9dVXhIWF8dprrzF9+nTs7OxISUkhOjqasWPH0rhxY8LCwvDy8sLJyYkff/yRYcOGGa61Y8cOhg8fXq73JIS4s0q1VpuonoY6NwUwjGozr2OM1vDa+ggW/hxF85ithO/5kdjYWExMTFi1ahU9evSgbdu2ADz77LMcOXIEPz8/3Nzc0Frz+uuvExcXh6WlJT4+PowfP74ib1EIkY8smSMqhaHOTQme3YfPRjuRnpFNQloGGjgX+SuBP/5MsnVPnnruBVxdXUlPT8fExASlFJAz6k1rbbiWUoqxY8fyzTffkJCQwKFDh3jiiScq6M6EEIXJE4+oVAqPcMu+mUqN2nVJy1LsjDhH6onDxbrO+PHjeeqpp6hduzajRo2iZk35X12IykL+NYpKpfBINtOWriSFbyfp6FaykuOp2+QxgAJPOADGxsYkJSUZ3ltZWWFlZcV7773Hjh07yr7iQohik8AjKpXCI9xUTWMaP52zNE7y8V3cOLKRV199tcCqBSNHjkQpxdSpUzE1NeXQoUOYmpri5eVFXFwcHTp0KPf7EELcmSr8zbEiubm5aUkEV73lrWJQeEJpnqbmpgTP7lOsa73yyis4OzszceLE0qyiEJWOUipMa+1W0fUoLnniEZVK3gi3uVtOkpCWUWBf/iHW9+Lq6krdunX597//Xep1FEKUjDzxiEor/8KhVuamzPRsZwhMQoj/kSceIUrJUOemEmiEeADJPB4hhBDlSgKPEEKIciWBRwghRLmSwCOEEKJcSeARQghRriTwCCGEKFcSeIQQQpQrCTxCCCHKlQQeIYQQ5UoCjxBCiHIlgUcIIUS5ksAjhBCiXEngEUIIUa4k8AghhChXEniEEEKUKwk8QgghypUEHlGm/vnPf7Jz586KroYQohKRDKSizGRlZTF//vxSv67WGq01NWrI9yYhqiL5lyvuS2xsLDY2Nnh7e+Pg4MDIkSNJTU3F2tqa+fPn061bNzZs2MC4ceMICAgAwNramjlz5tClSxfc3Nw4evQonp6etG7dmi+//BKA5ORk+vbti4uLC/b29mzevNlQXvv27XnppZdwcXHh3Xff5bXXXjPUZ+XKlfzjH/8o/w9CCPH35X17LOkPYASEA1tz37cEfgXOAusBk3tdw9XVVYvS9dlnn+mUlJRSv25MTIwG9IEDB7TWWo8fP14vXLhQt2jRQn/00UeG47y9vfWGDRu01lq3aNFCL126VGut9YwZM7S9vb2+ceOGvnr1qra0tNRaa52RkaETExO11lrHxcXp1q1b6+zsbB0TE6OVUvrQoUNaa62Tk5N1q1at9K1bt7TWWnfp0kVHRkaW+n0KURUAobqU/paXx09pPvG8CvyW7/1HwGda6zbAdWBiKZb1QNBak52dXaZlLFq0iNTU1FK5VmD4RTwW7Kbl7B8ZsewgFo9Y4eHhAcBzzz3HgQMHABg9evQdrzF48GAA7O3t6dy5M/Xr18fS0pLatWuTkJCA1po5c+bg4OBAv379uHjxIleuXAGgRYsWuLu7A1C3bl369OnD1q1bOX36NBkZGdjb25fKfQohylapBB6lVDNgILAq970C+gABuYesAYaWRllVXeEmo7Vr19KlSxdcXFwYNWoUycnJAISEhNC1a1ccHR3p1KkTSUlJZGVlMXPmTDp27IiDgwPLly8HYM+ePfTq1YuRI0diY2ODl5cXWmsWL17MpUuX6N27N7179y5RvQPDL/LWpuNcTEhDA1dupJOQmklg+EXDMTn/2XOCwp3UqlULgBo1ahhe573PzMzEz8+PuLg4wsLCiIiIoHHjxqSnpxd53UmTJuHr64uPjw/jx48v0f0JIcpPaT3xLALeBPK+vj8MJGitM3Pf/wk0LepEpdQUpVSoUio0Li6ulKpTuUVFRfH888+zY8cOVq9ezc6dOzl69Chubm58+umn3Lp1i9GjR/Of//yHY8eOsXPnTkxNTVm9ejVmZmaEhIQQEhLCypUriYmJASA8PJxFixZx6tQpoqOjCQ4OZvr06VhZWREUFERQUFCJ6rzw5yjSMrIKbMu8cZV/rtgEwLp16+jWrVuJygBITEykUaNGGBsbExQUxPnz5+94bOfOnblw4QLffvstY8aMKXHZQojyUeJRbUqpQcBVrXWYUqpX3uYiDtVFna+1XgGsAHBzcyvymKouMPwiC3+O4lJCGg11IpZNmuHu7s7WrVs5deqUobnq1q1bdOnShaioKJo0aULHjh0BeOihhwD45ZdfiIyMNHTWJyYmcvbsWUxMTOjUqRPNmjUDwMnJidjY2FIJBHkuJaTdts344ebEHN6Gg8NK2rRpw4svvsiSJUtKVI6XlxdPPfUUbm5uODk5YWNjc9fjn376aSIiImjQoEGJyr2TW7dukZGRcdenuOvXr5dZ+UI8iEpjOLUHMFgp9SRQG3iInCcgc6VUzdynnmbApVIoq8rJa6LKe1q4ciOdhIwaBIZfxEhr+vfvz7p16wqcExkZaWi2yk9rzZIlS/D09Cywfc+ePQWarYyMjMjMzCx8eolYmZtysXDwUQqH0TMJnt3HsCk2NrbAIb6+vkXuGzduHOPGjSty36FDh4qsw4kTJ27bduDAgQKj20rLb7/9xqpVq9i0aRObNm3C2dmZsLAw/vGPf5CcnIyFhQW+vr40adKEhQsXsnPnTiZOnMiYMWMMXxSEEEUrcVOb1votrXUzrbU18AywW2vtBQQBI3MP8wY2l7SsqqioJiqtNQt/jsLd3Z3g4GB+//13AFJTUzlz5gw2NjZcunSJkJAQAJKSksjMzMTT05Nly5aRkZEBwJkzZ0hJSblr+fXr1ycpKanE9zHTsx2mxkYFtimlmOnZrsTXvh8JCQm0bdsWU1NT+vbtWyrXTElJwcfHh27dujFp0iTat29PZGQkzs7OZGRkMG3aNAICAggLC2PChAm8/fbbAHzwwQesXbuW6OhoXFxcGDdunGGghRDidmU5gXQW4K+Ueo+cYdary7CsSquoJqq87ZaWlvj6+jJmzBhu3rwJwHvvvUfbtm1Zv34906ZNIy0tDVNTU3bu3MmkSZOIjY3FxcUFrTWWlpYEBgbetfwpU6bwxBNP0KRJkxL18wx1zumiy2sybNHCms9/DjZsL2/m5uacOXOmVK/ZpEkTHBwcWLVq1W1NfFFRUZw4cYL+/fsDOZNjmzRpYtjfrl07PvroIz744AP8/f0ZNGgQzz//PIsXLy7VOgrxIFA5Q8ArBzc3Nx0aGlrR1ShVHgt2395EBTQ1Ny3QRCUqRv7+tzpXT9LgzwNcu3CWMWPG4O3tTYsWLQA4fvw4U6ZMuWMzoNaaoKAgfHx8+PXXX3n66aeZNGkS1tbW5Xg3orpSSoVprd0quh7FJSsXlLGimqhMjY0qrIlK/E/hIeIpjWy51vkl3lq6ATMzM4YMGUK/fv2IjY2lXbt2xMXFGQJPRkYGJ0+eBMDPzw8bGxu++OILxowZw2+//cZ7770nQUeIO5C12spY4SYqK3NTZnq2q7AmKvE/RfW/pWVksfzXqwTPfpVXX32VI0eOYGRkhImJCQEBAUyfPp3ExEQyMzOZMWMGtra2tGjRgv3799OoUaMKuhMhqhZpahPVVsvZPxY5xl8BMQsGlnd1hLhv0tQmRBVhZW76t7YLIUqHBB5RbUn/mxAVQ/p4RLUl/W9CVAwJPKJaG+rcVAKNEOVMmtqEEEKUKwk8QgghypUEHiGEEOVKAo8QZahr164VXQUhKh0JPEKUoYMHD1Z0FYSodCTwCFGG6tWrB9w5PbkQ1ZEEHiHKSVHpyYWojmQejxClLH+qhbSMLALDL2IOZZ6eXIiqQgKPEKWocKpzreGtTcfxejSpzNOTC1FVSFObEKXoTqkW/EMuVFCNhKh8JPAIUYrulOr8r+Sb5VwTISovyccjRCmSVOeiIkg+HiGqMUm1IMS9yeACIUqRpFoQ4t4k8AhRyiTVghB3J01tQgghypUEHiGEEOVKAo8QQohyJYFHCCFEuZLAI4QQolyVOPAopZorpYKUUr8ppU4qpV7N3d5QKbVDKXU293eDkldXCCFEVVcaTzyZwOta6/aAO/CyUqoDMBvYpbVuA+zKfS+EEKKaK3Hg0Vpf1lofzX2dBPwGNAWGAGtyD1sDDC1pWaJ4Fi1aRGpq6gNTjhDiwVKqfTxKKWvAGfgVaKy1vgw5wQlodIdzpiilQpVSoXFxcaVZnUopKyvr3geVkAQeIURlVmqBRylVD9gIzNBa3yjueVrrFVprN621m6WlZWlVp0zc6w9tbGwsNjY2eHt74+DgwMiRI0lNTcXa2pr58+fTrVs3NmzYwLlz53j88cdxdXWle/funD59GoANGzZgZ2eHo6MjPXr0AHIC1cyZM+nYsSMODg4sX74cuHMq5cWLF3Pp0iV69+5N7969gZz0y7NmzcLV1ZV+/fpx5MgRevXqRatWrdiyZUuplSOEEMWitS7xD2AM/Az8I9+2KKBJ7usmQNS9ruPq6qorsxYtWui4uLg77o+JidGAPnDggNZa6/Hjx+uFCxfqFi1a6I8++shwXJ8+ffSZM2e01lofPnxY9+7dW2uttZ2dnf7zzz+11lpfv35da6318uXL9bvvvqu11jo9PV27urrq6OhoHRQUpB966CF94cIFnZWVpd3d3fX+/fuLrCegt23bprXWeujQobp///761q1bOiIiQjs6OpZaOUKIigGE6lL4W15eP6Uxqk0Bq4HftNaf5tu1BfDOfe0NbC5pWcXx9ddf4+DggKOjI2PHjuX8+fP07dsXBwcH+vbtyx9//AHAuHHjCAgIMJxXr149oPjf8FevXs1rr71GYPhFPBbsxuLxaXQa+SIWj1jh4eEBwHPPPceBAwcAGD16NADJyckcPHiQUaNG4eTkxAsvvMDly5cB8PDwYNy4caxcudLQJPfLL7/w9ddf4+TkROfOnbl27Rpnz54F/pdKuUaNGoZUykUxMTHh8ccfB8De3p6ePXtibGyMvb294ZzSKEcIIYqjNBYJ9QDGAseVUhG52+YAC4DvlFITgT+AUaVQ1l2dPHmS999/HzMzM44cOUJ8fDze3t48//zzeHt789VXXzF9+nQCAwPvep3w8HBOnjyJlVVOEAkODmb69Ol8+umnBAUFYWFhQUpKCk1btCQwpQ26YQuSj+/EyKwRN2+kEhh+0bBIZE5chrp16wKQnZ2Nubk5ERERBcqMiIhg8ODBPPzww/z444+0a9eOF198Ea01S5YswdPTs8Dxe/bsKXYqZWNjY0M9atSoYTivRo0ahnNKoxwhhCiO0hjVdkBrrbTWDlprp9yfbVrra1rrvlrrNrm/40ujwoXlPXG0nP0jT731JfbdPDly5AgADRs25NChQzz77LMAjB071vAEcjfF+YZft25dbmJC/LGdZFy7gM7OokFPb7JSE/jnik0ArFu3jm7duhU476GHHqJly5Zs2LAByPmDf+zYMSIiIli3bh2dO3dm/vz5NG/enBEjRuDp6cmyZcvIyMgA4MyZM6SkpNy1/vXr1ycpKeme95lfeZUjhBBVOi1CYPhF3tp03JDjPjH1FnuibmBapy5pqSns2bOHxMRERo8ezalTp3B2dgZg+/bt7Nu3jyeffBKAoKAgw6CBkJAQQkNDcXFxoXXr1jRo0IDMzExmz57NpUuX6NmzJ0888QTDhw/nZlI8N49uI/nEbuo7DyR+92qM6j1MzOFttGz5KX/99Rd2dnbEx8fz7LPP8ssvv3DkyBHS09OZOHEi48aNo0mTJnh5eeHj48OVK1cICAjAwsICGxsbVq5cyZIlSzh27JihHqampuzfvx+AY8eOMX36dEJDQzlx4gRaa8aNG8eUKVN44oknaNKkCUFBQcX6LCdNmkRsbCwuLi5orbG0tLznk+H9lCOEEBXeyZT/5+8OLuj64S7dYtZWw0+TCV/omg2sdA3j2lprrTdv3qxr1qypFy1apLOysnTr1q119+7ddUZGhjYzM9MzZszQWmvt6empAR0XF6cdHBy0p6en1lrrBQsW6E6dOuklS5botm3baltbWx0dHW3o+Ld0GaBrmjXWRvUtdLNX/bVp267aqL6Fdp+/XTdr1kxHR0drrbV+5pln9MCBA7XWWicmJuqMjAyttdY7duzQw4cP11pr7ePjo19++WXDveV/P2jQIO3r66u11nr16tV6yJAhWmutvb299ciRI3VWVpY+efKkbt269d/6/IQQDwaq2OCCKv3Ec6lQbnsTyxaYdRnNtW2LaNm2A+1aPUrnzp3ZvHkzq1evJj09neHDh1OzZk2eeuoptm7dyoEDBzhz5gxKKX7++WdiY2OJjY3FycmJW7duAWBqakrt2rWpU6cO3bp147HHHmPv3r3YNTXj1/gmZGVrjGrnDE7QGem0vLwLk1ataNmyJQBjxoxhxYoVACQmJuLt7c3Zs2dRShmatu7m0KFDbNqU03w3duxY3nzzTcO+oUOHUqNGDTp06MCVK1dK/qEKIUQZq9KLhFqZm962rZ59X5RxLWqP/pRuIyZibm7O7t27iYyMZOjQoZibmwM5o9rs7e15//336devH48++igATz75JImJiURERHDq1ClOnTrFxIkTOXLkCO+88w59+/bFyMgIgEcb1sEk6SJmjzRDkTMy7vX5nzBz8rN3rPM777xD7969OXHiBD/88APp6el/+77zBgoABTr+c774CCFE5ValA89Mz3aYGhsVuS8tIwv/kAtF7vvmm2+YPXs2W7duZerUqYwalTPgztXVleDgYP7973/TqVMnHBwceOaZZ0hMTCQxMZEaNWpw7Ngx9u3bR8+ePdm0aRPJidep+Wc4Zj+9jWujGsSG7OKnn34iOjqabdu24e7ujre3N6GhoVy/fp3ExESWL1/OrFmz8PDw4OLFi+zfv/+uHfVdu3bF398fAD8/v9sGLAghRFVSpQPPUOemfDjc/o77/0q+edu2S5cusX79eg4ePMiUKVO4cOGC4Q++hYUF8+bN49133yUtLacZ78aNG6xZswZPT08GDx5Meno6q1ev5vvvv2f79u2GJ6iNGzfSuHFjIGf48tKlSxk2bBgpKSmMHz8eS0tL5s2bx5tvvklMTAxr165lwoQJNGzYkHnz5tG7d29OnTqFk5MT69evL1DnxYsX4+Pjg4ODA2vXruU///lPqXx+QghREVRlap5xc3PToaGhf/s8jwW7uViovwegqbkpwbP7FNj2+eef88EHH9CoUc7ScWlpaYwZMwZfX19CQ0Px9/cvcr+rqyv+/v74+fkVuN7cuXOpV68eb7zxRoH3Y8aMoUuXLpw/f56XX36ZBg0asH37do4ePUqvXr14//338fDw4MqVK3h4ePD777//7fsWQggApVSY1tqtoutRXFX6iSdPUU1upsZGzPRsZ3ifN9/nX5tPUKNtL+b6/EhERARRUVHMnTvXcJzWGm9vbyIiIgrs11oX6Fu5lzVr1vDf//4XW1tbEhMTGTNmTIH9eX0zMiFTCFHdPBCBJ6/Jram5KYqcJ50Ph9sbVg/Im+9zMSGNWi0cuRK5h5lr9xMYfpH4+HjOnz9vuFbfvn0JCAjg6tWrAIb9Xbp0Ye/evcTExBi2w50nUc6ZM4f27duzfPly/Pz8CAgIoGfPnmX8SQghROVXpYdT5zfUuakh0BS28OcowyRTE4tHMe8+lvN+c/BaBzZWDfjiiy8Mx3bo0IH33nuPAQMGkJ2djbGxMV988QXu7u6sWLGC4cOHk52dTaNGjdixYwdPPfUUI0eOZPPmzSxZsqRAuWvWrGHq1KmkpqbSqlUrfHx8yu4DEEKIKuKB6OO5l5azf6Sou1RAzIKBpV6eEEKUJ+njqYSKmu9zt+2lqVevXpRFMBVCiKqqWgSe4gw+EEIIUT6qReC51+CD0nCn7KP5vfjii7i5uWFra8u//vUvw/aQkBC6du2Ko6MjnTp1Iikp6Y4ZQYUQoqp7YAYX3MvdBh+UlqioKFavXo2HhwcTJkxg6dKlBfa///77NGzYkKysLPr27UtkZCQ2NjaMHj2a9evX07FjR27cuIGpqSmrV6/GzMyMkJAQbt68iYeHBwMGDDCs/yaEEFVVtQk8ZSEw/CILf47iUkIaDXXibdlHFy9eXOD47777jhUrVpCZmcnly5c5deoUSimaNGlCx44dgZx8PZCTETQyMtKQJTUxMZGzZ89K4BFCVHnVoqmtLOSfG6SBKzfSSUjNJDD8ouGY/BNOY2Ji+OSTT9i1axeRkZEMHDiQ9PT0O05M1bkZQfMmssbExDBgwIB71mvo0KG4urpia2trWBG7Xr16vP322zg6OuLu7s6VK1dISkqiZcuWhtWxb9y4gbW1dbFWyxZCiJKQwHOf8s8NypN54+ods4/euHGDunXrYmZmxpUrV9i+fTsANjY2XLp0iZCQEACSkpLIzMy8r4ygAF999RVhYWGEhoayePFirl27RkpKCu7u7hw7dowePXqwcuVK6tevT69evfjxxx8B8Pf3Z8SIERgbG5f8wxFCiLuQwHOfCucCAjB+uDkxh7fh4OBAfHw8L774omGfo6Mjzs7O2NraMmHCBEOTnImJCevXr2fatGk4OjrSv39/0tPTmTRpEh06dMDFxQU7OzteeOGFYi2ts3jxYsOTzYULFzh79iwmJiYMGjQIyFmBOy+V96RJkwyTWn18fBg/fnxJPxYhhLgn6eO5T1bmprcvTKoUDqNnFliYdM+ePYbXvr6+RV6rY8eOHD58+LbtH3zwAR988MFd65G/n6lufBRZIdsIO3SIOnXq0KtXL9LT0zE2NjY05+VfG87Dw4PY2Fj27t1LVlYWdnZ2xbhzIYQoGXniuU9FzQ1SSpXr3KDC/UxXr13nQoril6jrnD59ushgVtjzzz/PmDFj5GlHCFFuJPDcp8Jzg1q0sGb9z8FlPmQ7v8L9TKYtXcnKzMLrye688847uLu73/MaXl5eXL9+/bbVs4UQoqxUi7XaHlSlsQZdQEAAmzdvZu3ataVaNyFE+alqa7VJH08VVmQ/E8Vfg27atGls376dbdu2lXbVhBDijqSprQor6Rp0S5Ys4ffff6dt27ZlUT0hhCiSPPFUYXn9SXmj2qzMTZnp2a5c+5mEEOLvkieeKm6oc1OCZ/chaKotid9Ml6AjRBXXtWvXiq5CmSvzwKOUelwpFaWU+l0pNbusyxNCiKrs4MGDxT42Kyvr3gdVQmUaeJRSRsAXwBNAB2CMUqpDWZZZnWVlZTF58mRsbW0ZMGAAaWlprFy5ko4dO+Lo6MiIESNITU0lMTERa2trsrOzAUhNTaV58+ZkZGRw7tw5Hn/8cVxdXenevTunT58uUZ0SEhIMq3Tv2bPHsIJCcfn6+nLp0qUS1UGIqqRevXq3/Vt55ZVXDBPQra2tmT9/Pt26dWPBggW4uLgYjlNKtVFKhZV3nf+usn7i6QT8rrWO1lrfAvyBIWVcZrV19uxZXn75ZU6ePIm5uTkbN25k+PDhhISEcOzYMdq3b29It+Do6MjevXsB+OGHH+jatSvOzs5MmTKFJUuWEBYWxieffMJLL71UojrlDzz3QwKPELerXbs2Bw4c4O2338bMzAwgbyjreMC3wipWTGU9uKApcCHf+z+BzmVcZrVROC1DI6vmODk5Af9bk+3YsWP861//IiEhgeTkZDw9PQEMOYB69+6Nv78/w4cPJzIykoMHDzJq1ChDGTdv3ixRHWfPns25c+dwcnLC2NiYunXrMnLkSE6cOIGrqyvffPMNSinmz5/PDz/8QFpaGl27dmX58uVs3LiR0NBQvLy8MDU15dChQ5ialn26ciqHWSgAACAASURBVCHKU/5/x1bmpmRl33tu5ejRow2vJ02axJ49eyxyW5hGk/OFv1Ir6yee29f7p+CcR6XUFKVUqFIqNC4uroyr8+AIDL/I61/t5MhCb+J+/JTIVW9y5cpV1h/8HWtra3bv3s2KFSt45plneOWVV6hbty6ZmZls27aN69evM3jwYDZv3oytrS3btm0jIiICAHNzc2bMmEG3bt2IiIjgt99+Y9CgQYY153766SdcXFxwdHSkb9++AKSkpDBhwgQ6duyIs7MzmzdvNtRzwYIFtG7dmoiICBYuXEh4eDiLFi3i1KlTREdHExwcDOQ0JYSEhHDixAnS0tLYunUrI0eOxM3NDT8/PyIiIiToiAdO4WWvLiakcTMzm4PR1w1N4QDp6ekFzqtbt67h9YgRIwDMgEFAmNb6WnnUvSTKOvD8CTTP974ZUKDdRGu9QmvtprV2s7S0LOPqPDgW/hzFzcwsMuP/pJ7j4zR+ej5aKWa99wkANWvWZMKECWRnZzNv3jzef/99bGxsaNCgAfPmzaNevXqkpqbStGlTJk6cSI0aNTAyMqJly5aGFA1aa44dO2YoMy4ujsmTJ7Nx40aOHTvGhg0bgJzMqn369CEkJISgoCBmzpyJ/8GzeCzYTbePdhP9V4ohT1GnTp1o1qwZNWrUwMnJybBSdlBQEJ07d8be3p7du3dz8uTJcvw0hagYRaVXAdh05ianTp3i5s2bJCYmsmvXrjteo3bt2gCJwDLAp6zqWprKOvCEAG2UUi2VUibAM8CWMi7zgRYYfhGPBbsNKxYY1bekdrOc8Ro1atfl6u85gSKvyW3OnDmcOXOGd999FxsbGx577DH27dtHYmIixsbG7Nixg9GjRzN27FgA/Pz82LdvH/7+/tja2hZ4ejl8+DA9evQwZEFt2LAhkJMtdcGCBTg5OdGrVy+uJ6UwZ22QoY6ZWdm8tek4B87GUatWLcP18lbKTk9P56WXXiIgIIDjx48zefLk277hCfEgKiq9Ckpxjfo8/fTTODg44OXlhbOz870uFU9Oa9IvpV/L0lemfTxa60yl1CvAz4AR8JXWWr7K3qe8x/IC35ByGzNrmjXm4SdeJTNyG9yC1157DQsLCxITE1m0aJGhqezcuXOMGjUKrTV169blr7/+AiAyMhKAli1bMmvWLA4ePGgYFNCvXz+Au2ZL3bhxI+3a5ayYkD8wKhNTsm+lkZaRhX/IBayLuK+8IGNhYUFycjIBAQGMHDkSgPr165OUlHTfn5kQlVnhZa+y0m5Qo3Y9rMxN+Xj2x3z88ce3nZPXSlBIPXL+vlaJ8dVlPo9Ha71Na91Wa91aa/1+WZf3ICvqsTzrRhw3L/4GwM3T+xns2afAfjMzMxo0aMD+/fsBWLt2LT179sTc3BwzMzMOHDgA5Dzp5LG2tiYiIoLs7GwuXLjAkSNHAOjSpQt79+4lJiYGgPj4eAA8PT1ZsmQJeQvOxkSdMFzLyPQhajXtwKXVL3H2hy+LvC9zc3MmT56Mvb09Q4cOpWPHjoZ948aNY+rUqTg5OZGWVsS3QyGqsPzLXmUmXeO/a9/g4S4j/lZ6lWHDhgE8DPynTCpZBmR16iqk8GrUmYlXuLphLrWa25L93ygcOtiw64cAOnToQGhoKBYWFgBEREQwdepUUlNTadWqFT4+PjRo0ICwsDAmTJhAnTp18PT0JCAggBMnTqC15rnnniMiIgI7OzuuXLnC3Llz6dWrF9u3b2fOnDlkZ2fTqFEjduzYQVpaGjNmzODgwYNorbmUWY+Hhr5zW/2bmpsWSJInhLh9VNv9LHtV1VanlsBTheRvwoLcwBMwj46v+1SqP+hFNQmaGhvx4XB7WdJHiDJQ1QKPrNVWhVSGrKfFUThJXlNzUwk6QggDWZ26Cim8GnWLFtZ8Xs5ZT4trqHPTSlkvIUTFk8BTxcgfdCFEVSdNbUIIIcqVBB4hhBDlSgKPEEKIciWBRwhRoVJSUhg4cCCOjo7Y2dmxfv165s+fT8eOHbGzs2PKlClorTl37lyB3DNnz57F1dW1Amsu7pcEHiFEhfrpp5+wsrLi2LFjnDhxgscff9ywWvmBAweIiIhg69attG7dGjMzM8NK6j4+PowbN47Y2Fjs7Owq+C7E3yGBRwhR7vIWu205+0feP5jMlm0/M2vWLPbv34+ZmZlhtfLOnTsTHh5uWK180qRJ+Pj4kJWVxfr163n22Wcr+E7E/ZDAI4QoV4Vz0MQbW2A25t/crN+Ut956i/nz5xtWK89blXnRokW89tprrFq1iuXLl9OqVSseeeQRHn744QLXjo6OxtnZmZCQELKyspg5cyYdO3bEwcGB5cuXV8DdiqJI4BFClKvCi91mJl3jJjUJqWnHG2+8wdGjR4Gc1crfeecdjIyMmDp1KgsXLmTz5s1MnjyZmzdvEhMTQ/4lv6KiohgxYgQ+Pj507NjRkOY9JCSEkJAQVq5caVjgVlQsmUAqhChXhXPQZMTFcnWPD5eV4v1HH2bZsmUEBgZib29P48aNDZlntdbMmTOHn376iWvXrlGzZk2uXLkC5CQpHDJkCBs3bsTW1hbIyRMVGRlJQEAAAImJiZw9e9aQT0pUHAk8QohyVTgHjWkrV0xbuRZYvfxPoyYE1evD+fOx3Dz3Lk5DJuPn52fIgpucnMzatWsNuZzMzMxo3rw5wcHBhsCjtWbJkiV4enqW/01WIUopN+B5rfV0pVQv4JbW+mDuvqHAGa31qdIsU5rahBDlqqjFbk2NjQyL3ebvA1ImptxKS+GtTcfZf/I8ISEh+Pn54erqyvnz5w3nm5iYEBgYyNdff823334L5OSJWrZsGRkZGQCcOXOGlJSUcrrLqkNrHaq1np77thfQNd/uoUCH0i5TnniEEOWq8GK3hXPQ5O8DykskeO7LF7jWoj2tGzcmIyODrVu3YmNjU+C6devWZevWrfTv35+6desyadIkYmNjcXFxQWuNpaUlgYGB5Xuzf9PXX3/NJ598glIKBwcH3nvvPSZMmEBcXByWlpb4+Pjw6KOPMm7cOExNTTl9+nReAK6nlPoK6AL8qrUeB6CUSga+APoB14E5wMfAo8AMrfWW3KecN4BXgKlAllLqOeBVYDDQUyn1f8CI3Gp+AVgCqcBkrfVppZQl8GXudcm9dvCd7lPy8QghKpXCCQ/zKCBmwcDyrk65OXnyJMOHDyc4OBgLCwvi4+Px9vZm5MiReHt789VXX7FlyxYCAwMZN24c6enprFu3ji1btjB06NBswAk4CYQAE7XWEUopDTyptd6ulPoeqAsMJOcpZo3W2ikv8GitByml5gLJWutPAJRSvsBWrXVA7vtdwFSt9VmlVGfgQ611H6XUt8BSrfUBpdSjwM9a6/Z3uld54hFCVCqF+4Dyb3/Q5M8+qk79hEs3T0Pm4IYNG3Lo0CE2bdoEwNixY3nzzTcN5z711FMopbC3twfI0FofB1BKnQSsgQjgFvBT7inHgZta6wyl1PHcY4pNKVWPnGa4DUqpvM21cn/3Azrk2/6QUqq+1jqpqGtJH48QolK5Vx/Qg6LwfKaE1FvsiYojMPziHc/J94edWrVy/ubXqFEDKPCQmM3/Hioy9P+atbKBmwBa6/zHFFcNIEFr7ZTvp32+fV3ybW96p6CTd7AQQlQa1SWDbeH5TLVbOJJ4ah8fbDoCQHx8PF27dsXf3x8APz8/unXrVtbVSgLqF/Vea30DiFFKjQJQORxzj/uFnD4icvc53a0QaWoTQlQ61SHhYeH5TCaWLTDrMpqIL2fguGUuzs7OLF68mAkTJrBw4ULD4IIy9gMQoJQaAkwD/IGVSqnpwEjAC1iWO9jAOHf/MWA68IVSKpKcuLKPnIEKRZLBBUIIUQE8Fuwusi8r/3ym4lJKhWmt3UqrbmVNmtqEEKICVJe+rKJIU5sQQlSAe81nepBJ4BFCiApSHfqyiiJNbUIIIcqVBB4hhBDlqkSBRym1UCl1WikVqZT6Xillnm/fW0qp35VSUUopWR5WCCEEUPInnh2AndbaATgDvAWglOoAPAPYAo8DS5VSRne8ihBCiGqjRIFHa/2L1joz9+1hoFnu6yGAv9b6ptY6Bvgd6FSSsh4Uc+fO5ZNPPqnoagghRIUpzT6eCcD23NdNgQv59v2Zu+02SqkpSqlQpVRoXFxcKVan9GVmZt77ICGEEHd1z+HUSqmdwCNF7Hpba70595i3gUzAL++0Io4vcokErfUKYAXkrFxQjDqXmXfffRc/Pz+aN2+OhYUFrq6ubN26la5duxIcHMzgwYNp27Yt7733Hrdu3eLhhx/Gz8+Pxo0bM3fuXP744w+io6P5448/mDFjBtOn5+RWev/99/n6669p3rw5lpaWuLq6VuRtCiFEhbpn4NFa97vbfqWUNzAI6JtvFdQ/geb5DmsGXLrfSpaH0NBQNm7cSHh4OJmZmbi4uBgCREJCAnv37gXg+vXrHD58GKUUq1at4uOPP+bf//43AKdPnyYoKIikpCTatWvHiy++SGRkJP7+/kVeVwghqqMSTSBVSj0OzAJ6aq1T8+3aAnyrlPoUsALaAEdKUlZZyJ8LgxPb6NSpN6amOTk/nnrqKcNxo0ePNrz+888/GT16NJcvX+bWrVu0bNnSsG/gwIHUqlWLWrVq0ahRI65cucL+/fsZNmwYderUAWDw4MHldHdClK6uXbty8ODB27aPGzeOQYMGMXLkyAqolaiKStrH8zk5S2bvUEpFKKW+BNBanwS+A06Rk4ToZa111p0vU/4K58JITLvFrtNXi8yFUbduXcPradOm8corr3D8+HGWL19Oenq6YV9efgwAIyMjQ59Q/hwaQlRVRQUdIe5HSUe1Paa1bp4v+c/UfPve11q31lq301pvv9t1KkLhXBi1mnUg6cyvfLT1OMnJyfz4449FnpeYmEjTpjnjJNasWXPPcnr06MH3339PWloaSUlJ/PDDD6VzA0KUgm+++YZOnTrh5OTECy+8wBdffFEgy6Wvry/Tpk0DoF69egBorXnllVfo0KEDAwcO5OrVq4bjw8LC6NmzJ66urnh6enL58mUAevXqxaxZs+jUqRNt27Zl//79AGRlZfHGG29gb2+Pg4MDS5Ysuet1xIOh2q5cUDgXRq0mbTF9rBOhiyYxfPhw3NzcMDMzu+28uXPnMmrUKLp3725IUXs3Li4ujB49GicnJ0aMGEH37t1L7R6EKInffvuN9evXExwcTEREBEZGRtSrV8+Qahlg/fr1BZqaAb7//nuioqI4fvw4K1euNDwJZWRkMG3aNAICAggLC2PChAm8/fbbhvMyMzM5cuQIixYtYt68eQCsWLGCmJgYwsPDiYyMxMvL657XEVVftV0ktKi87g91Go7toEkETnenR48evP7660yePLnAMUOGDGHIkCG3XW/u3LkF3p84ccLw+u2335Z/OKJSyN+vqU79TMLhEDp27AhAWloajRo1olWrVhw+fJg2bdoQFRWFh4dHgWvs27ePMWPGYGRkhJWVFX365OSOiYqK4sSJE/Tv3x/IeZpp0qSJ4bzhw4cD4OrqSmxsLAA7d+5k6tSp1KyZ86eoYcOGnDhx4q7XKY569eqRnJx82/Yvv/ySOnXq8Pzzzxd53p49e/jkk0/YunXr3ypP/D3VNvDM9GzHW5uOF2huS/zlC4wyruLim4W3tzcuLi4VWEMhSldev2be//OJabdQbXsy9/NPC6yQvHr1ar777jtsbGwYNmxYkX2URW3TWmNra8uhQ4eKLD+vDzR//6fW+rZr3es6JTF16h2TYla4hIQEvv32W1566aUHPgBW26a2ovK6r1n7DTFRJzl9+jRvvfVWRVdRiFJVuF+zdgtHbvy2n/c3HgYgPj6e8+fPM3z4cAIDA1m3bt1tzWyQ02/p7+9PVlYWly9fJigoCIB27doRFxdnCBgZGRmcPHnyrnUaMGAAX375pSEQxcfHF+s6H3/8MYsXLwbgtddeMzx17dq1i+eeew7IaWlwdHTE3d2dK1euAAVXDvn999/p168fjo6OuLi4cO7cOQCSk5MZOXIkNjY2eHl5UV5ZmhMSEli6dGm5lFXRqm3ggZzgEzy7DzELBhI8u0+1zIshqo/C/ZomFo9i3n0sx1bOxMHBgf79+3P58mUaNGhAhw4dOH/+PJ063b7S1bBhw2jTpg329va8+OKL9OzZM+d6JiYEBAQwa9YsHB0dcXJyuudIuEmTJvHoo4/i4OCAo6Mj3377bbGu06NHD8MAhdDQUJKTk8nIyODAgQN0796dlJQU3N3dOXbsGD169GDlypW3le3l5cXLL7/MsWPHOHjwoKE5Lzw8nEWLFnHq1Cmio6MJDg4u/odcArNnz+bcuXM4OTkxc+bMOwbAB2HgRbVtahOiuimqX7Nu+x607eJJ8Ow+BbYX1cST12eilOLzzz8vsgwnJyf27dt32/Y9e/YYXltYWBj6eGrWrMmnn37Kp59+Wqzr5PVRXbyWxH93BbPuQBS1atXCxcWF0NBQ9u/fz+LFizExMWHQoEFATp/Sjh07ClwnKSmJixcvMmzYMABq165t2NepUyeaNWtmqEdsbCzdunUr8n5L04IFCzhx4gQRERHs2bOHIUOGcPLkSaysrPDw8CA4OJjOnTszbdo0Nm/ejKWlJevXr6+S/ccSeISoJorq1zQ1NmKmZ7sKrFXxFeijMqoJ9S2Z8e4ierayo3v3bgQFBXHu3Dnat2+PsbGxoe8of59Snrs1n91pPl5ht27dIiMjo8A8v/yuX79OgwYNinVfC3+O4vz5WOL/SiEw/CLmFB0Azc3NSzzwojKo1k1tQlQnRfVrfjjcvso0Md/WR9XclmuHNnIyuyndu3fnyy+/xMnJqVgTth966CGaNWtGYGAgADdv3iQ1NfUeZ+X47bffeP3112nXrh1nzpwBcprJOnTogIODA2+88QaQMxTdzs6OTz75hDstgJx/IjtAZlY2b206zoGzcUUGwLyBFxEREURERHD8+HF++eWXYtW7MpEnHiGqkaHOTatMoCnstrl3zWxJPPQdyQ+1onHjxtSuXftvzZNbu3YtL7zwAv/85z8xNjZmw4YNdzw2JSWF7777jtWrV6O1Zvz48URGRlK/fn3i4+P5/vvvOX36NEopEhISgJwRdAMHDsTX15cePXpga2vLpEmTGDBgADVq5Hznzx9MlYkp2bfSSMvIwj/kAtZF1CP/wIsuXbqQkZFhCH5ViSqvERvF4ebmpkNDQyu6GkKISshjwe7b+qgg58mtcB9Vacg/5+nCoqdp096W79d9jY2NTYHjMjMzcXV1xc3NjYEDBzJo0CBMTEwKHKO15qeffmLSpEm4urqyZcsWAFrO/rHAsv1xWxaSEReDqlmL/q5tDX1tr7zyCm5ubowbN46IiAimT59OYmIimZmZzJgxgylTpoRprd1K/UMoIxJ4hBBVQuF5SJDTR1UWzYWFy0qLOUraiZ08lHqJKePH4u3tTYsWLQzH37x5k127duHv78+ff/7J7t27DfuOHDmCj48PO3bsYNCgQUyePBlbW1ug9IKpUqpKBR5pahNCVAl5wSXvKcTK3JSZnu3KpOmwcH+SaUsXTFu60Mj4FmZmfzBkyBAsLCxYtWoVFhYWpKam8uSTT+Lu7s5jjz0GwC+//MIbb7zBI488wsSJE/nPf/5z25NQVR/wcb8k8Aghqozy6qMq3J+UJy7DhFdffZVXX32VI0eOYGRkRFJSEkOGDCE9PR2tNZ999hkADz/8MD/88EOBJ6PCyjOYViYSeISoZvIvzQJw6dIlpk+fTkBAQAXXrPIoas5T3vY8+SfXHjlye7qx4iZ8rMoDPu6XDKcWopopvDSLlZWVBJ1CZnq2w9TYqMC26tAEVl4k8AhRwQrnxDl//jxt2rThr7/+Ijs7m+7duxvmahQ+Nisrp2/gp59+wsXFBUdHR/r27QsUXJcMwM7OjtjY2NuWZomNjcXOzg6Azp07F1gXrVevXoSFhZGSksKECRPo2LEjzs7ObN68ubw+ngpR1ec8VXbS1CZEBcqfE8fY2JiXXnqJvXv3MmvWLKZOnUrnzp3p0KEDAwYMKPJYPz8/nnjiCSZPnsy+ffto2bIl8fHxdy0z/9IsgGH5GoBnnnmG7777jnnz5nH58mUuXbqEq6src+bMoU+fPnz11VckJCTQqVMn+vXrd8dZ+w+C6tgEVl4k8AhRAfLmiJze9R1Jvx6irZ0TZqbGhpw4c+fOZcOGDXz55ZeGALFr1y7CwsJuy59z+PBhevToQcuWLYGcnDb36+mnn6Z///7MmzeP7777jlGjRgE5I7S2bNlieIJKT0/njz/+oH379iX5GEQ1JYFHiHKWf46IBkxte1O730Tm5mvKSU1N5c8//wRyFuesX78+Wmu8vb358MMPC1xvy5YtRS4TU7NmTbKzsw3v09PT71m3pk2b8vDDDxMZGcn69etZvnw5kDMBcuPGjbRr9+D0cSxevJhly5bh4uKCn59fRVenWpE+HiHKWf45IrVbOJIaFUxywjUW/hxlyIkza9YsvLy8mD9/viELbt++fQkICODq1avA//LndOnShb179xITE2PYDmBtbc3Ro0cBOHr0qGF//fr1SUpKumP9nnnmGT7++GMSExOxt7cHwNPTkyVLlhgW1wwPDy/tj6XcLV26lG3bthUr6NxpoVBxfyTwCFHO8s8RycuJc+W7dwj5dCL9+/cnNjaWkJAQQ/AxMTHBx8eHDh068N577zFgwIAC+XMsLS1ZsWIFw4cPx9HR0ZC8bcSIEcTHx+Pk5MSyZcto27YtkDO/xMPDAzs7O2bOnHlb/UaOHIm/vz9PP/20Yds777xDRkYGDg4O2NnZ8c4775Txp1S2pk6dSnR0NIMHD+ajjz6ia9euODs707VrV6KiogDw9fVl1KhRPPXUUwwYMKCCa/xgkSVzhChn5b3mmCiatbU1oaGhmJiYUKdOHWrWrMnOnTtZtmwZGzduxNfXl//7v/8jMjKyRP1m5UGWzBFC3FV1XSalMsi/8Od/E9PZFnmZ3m3M8fb25uzZsyilyMjIMBzfv3//Sh90qiIJPEKUs+q6TEpFK7zwZ2a25t0fT+F7diND+/Xm+++/JzY2ll69ehnOeZCHi1ckCTxCVACZI1L+Ci/8CZCekcWxc5d42Tvnv4Wvr28F1Kz6kcEFQohq4U4Lf5q4DOWtt97Cw8PDsBKEKFsyuEAIUS08yIM6qtrgglJ54lFKvaGU0kopi9z3Sim1WCn1u1IqUinlUhrlCCHE/ZKFPyuPEvfxKKWaA/2BP/JtfgJok/vTGViW+1sIISqEDOqoPEpjcMFnwJtA/uVqhwBf65x2vMNKKXOlVBOt9eVSKE8IIe6LDOqoHErU1KaUGgxc1FofK7SrKXAh3/s/c7cVdY0pSqlQpVRoXFxcSaojhBCiCrhn4FFK7VRKnSjiZwjwNvDPok4rYluRoxi01iu01m5aazdLS8u/V3shhCihxYsX0759e7y8vCq6KtXGPZvatNb9itqulLIHWgLHclfGbQYcVUp1IucJp3m+w5sBl0pcWyGEKGVLly5l+/bthrQSouzdd1Ob1vq41rqR1tpaa21NTrBx0Vr/F9gCPJ87us0dSJT+HSFERfv000+xs7PDzs6ORYsWFVgs9LPPPqvo6lUbZTWBdBsQDfwOrAReKqNyhBAPmK5du5bJdcPCwvDx8eHXX3/l8OHDrFy5khdeeAErKyuCgoLua8XtJ598koSEBBISEli6dGkZ1PrBVGqBJ/fJ56/c11pr/bLWurXW2l5rLbNChRDFcvDgwdu2FV5RQGtdIMnd3QSGX8RjwW4GzFpOQiMndpxJoF69egwfPpz9+/ffVx3zyt+2bRvm5uYSeP4mWTJHCFGp1KtXD4A9e/bQu3dvnn32Wezt7YmNjaV9+/a89NJLuLi4cOHCBX755Re6dOmCi4sLo0aNIjk5GYBt27ZhY2NDe+dOTHjhJcJXzUZriDsTzpSZ/yIw/CIAH3744W1J3pKTk+nbty8uLi7Y29uzeXPOTJGiyre2tuavv/5i9uzZnDt3DicnJ2bOnMnYsWMN5wF4eXmxZcuW8vj4qgatdaX5cXV11UKI6q1u3bpaa62DgoJ0nTp1dHR0tNZa65iYGK2U0ocOHdJaax0XF6e7d++uk5OTtdZaL1iwQM+bN0+npaXpZs2a6ejoaN31w126Tvse2rR1R/2I9yJdo46ZNusxVneet1Xb2trq1q1b66ZNm+q4uDhDuRkZGToxMdFQRuvWrXV2dvZt5WutdYsWLXRcXJyOiYnRtra2hu179uzRQ4YM0VprnZCQoK2trXVGRkaZfWZAqK4Ef8OL+yOrUwshKlz+PDlpGVkEhl/EHOjUqVOB0WYtWrTA3d0dgMOHD3Pq1Ck8PDwAuHXrFl26dOH06dO0atWKli1bcinhFHXb9yT52E/UeuQxTBq1Iin0B46e2svHb7/GqlWrSE9PL1AXrTVz5sxh37591KhRg4sXL3LlypXbyr+bnj178vLLL3P16lU2bdrEiBEjqFlT/tzmkU9CCFGhCufJ0Rre2nQcr0eTbsuHk/+91pr+/fuzbt26AseEh4cbXluZm3I2375azW2pbe1IB8+xzJjRh88//5wDBw5gYWFhOMbPz4+4uDjCwsIwNjbG2traEJz+Tn6esWPH4ufnh7+/P1999VWxz6sOpI9HCFGhisqTk5aRhX/IhTuckcPd3Z3g4GB+//13AFJTUzlz5gw2NjZER0cTGxvLTM923Iz63wCCmmaNyboazUzPdhw9epSYmJjbrpuYmEijRo0wNjYmKCiI8+fP3/Me6tevT1JSUoFt48aNY9GiRQDY2tre8xrViTzxCCEq1J3y5PyVfBPru5xnaWmJr68vY8aM4f/bu//Yqu4yjuPvhyodCT8mMHEts6vYppTSXQwUDQ2IM6zdMCTM4gAABt9JREFUytomJY5gUsQ/ILkhG9HogH/4o5oAjTakEiJqMpIl08hkZMmM65zIH906BKL8cEgm4BBcVyqlIQIXHv+4p90VC1xCe8798XklpOfcc+7pc57c8NzzPd+n59q1awC0tbVRXl7Ozp07qaurY/r06SyJVdJ97AMM+GLNk/zn4/fY8q1nWLBgAeXl5f933FWrVrF8+XLmz59PLBajoqLinucwbdo0Fi1aRFVVFfX19Wzfvp0ZM2Ywe/Zsmpqa0shCftHzeETy3OrVq2loaKClpSWS3z8Wz8kZHBxk4sSJuDvxeJyysjI2bNjwoKHel6tXrzJ37lwOHz7MlClTxvR35eXzeEQkP/l99NPcyVg8J2f37t3EYjHmzJnD5cuXWbt27QPFeL+6urqoqKhg/fr1Y150spGueETyzJ49e2hvb8fMqK6upqCggMmTJ3Po0CEuXrzItm3baGlpYXBwkMbGRvr7+7lx4wZtbW00NjZy5swZ6uvrWbp0Kd3d3ezbt4+uri62bt1KUVERZWVlFBYW0tnZSW9vL+vWrePcueTjujo6OoZnoaVKndWm5+Tcv2y74ol8PnfqP/XxiIytY8eOeXl5uff29rq7e19fn7e2tnpLS4vfvHnTjx8/7rNmzXL39PtZzp8/7yUlJd7X1+fXr1/32tpaj8fj7u6+cuVKP3jwoLu7nz171isqKsI+5byA+nhEJJOkXk3Yid/ypdqnhqcPT506FYCmpibGjRtHZWXlcM+Kp9nP0tPTw5IlS4aPtWLFCk6dOgUkh5xOnDgxHMvAwABXrlxh0qRJ4Zy8ZCQVHpEcdnuPzOWr1/nD+wPsO3L+f4ayCgsLh5c9GH5Pt59laP+R3Lp1i+7ubiZMmDCq5yXZTZMLRHLY7T0yD5U8weUTf+SHr/YAcOnSpTu+N91+lpqaGg4cOEB/fz+JRIK9e/cOb1u2bBmdnZ3D60ePHn3QU5IcoCsekRx2e4/M+EdKmPKVb3B01ws8sX8L8+bNu+N70+1nKS4uZtOmTSxcuJCioiIqKyuHZ3Lt2LGDeDxOdXU1iUSCxYsXs2vXrtE7QclKmtUmksPGokdmJEN9M4lEgubmZtasWUNzc/OoHV/uLttmtWmoTSSHjUWPzEi2bNlCLBajqqqK0tJSdevLXWmoTSSHDU0gGOsemfb29lE9nuQ2FR6RHNc0r1jNmJJRNNQmIiKhUuEREZFQqfCIiEioVHhERCRUKjwiIhKqjGogNbNe4N7Pmc1+04GPow4iCyhP6VOu0pOreSpx90eiDiJdGVV48oWZHcqmLuOoKE/pU67SozxlBg21iYhIqFR4REQkVCo80fhp1AFkCeUpfcpVepSnDKB7PCIiEipd8YiISKhUeEREJFQqPCEys+1m9lcz+7OZ/cbMHk7ZttHMTpvZ+2b2VJRxZgIzqwtycdrMXow6nkxhZo+Z2dtmdtLMjpvZ88HrU83sTTP7W/DzM1HHmgnMrMDMjpjZ68F6qZm9G+Tpl2Y2PuoY85EKT7jeBKrcvRo4BWwEMLNK4DlgDlAH7DSzgjseJccF5/4ToB6oBFYGORJIAN9x99nAl4F4kJsXgbfcvQx4K1gXeB44mbK+FfhxkKd+4NuRRJXnVHhC5O6/c/dEsPoOMDNYbgRecfdr7v534DRQE0WMGaIGOO3uH7j7deAVkjnKe+5+wd0PB8tXSP6nWkwyPy8Fu70E5P0jQM1sJvAM8LNg3YCvAb8OdlGeIqLCE501wBvBcjHwj5RtHwav5SvlIw1m9jgwD3gXmOHuFyBZnIDPRhdZxugAvgfcCtanAf9O+fKnz1VE9ATSUWZmXcDnRti02d1fC/bZTHLI5OWht42wfz7Pc1c+7sHMJgJ7gRfcfSD5ZV6GmFkD8JG7/8nMvjr08gi76nMVARWeUebuX7/bdjNrBRqAJ/2TJqoPgcdSdpsJ/HNsIswKysddmNmnSRadl9391eDlf5nZo+5+wcweBT6KLsKMsAh41syeBh4CJpO8AnrYzD4VXPXocxURDbWFyMzqgO8Dz7r71ZRN+4HnzKzQzEqBMqAnihgzxHtAWTADaTzJiRf7I44pIwT3KX4OnHT3H6Vs2g+0BsutwGthx5ZJ3H2ju89098dJfn5+7+6rgLeBlmC3vM9TVPSXC0JkZqeBQqAveOkdd18XbNtM8r5PguTwyRsjHyU/BN9UO4AC4Bfu/oOIQ8oIZlYLHAT+wif3LjaRvM/zK+DzwDlghbtfiiTIDBMMtX3X3RvM7AskJ6tMBY4A33T3a1HGl49UeEREJFQaahMRkVCp8IiISKhUeEREJFQqPCIiEioVHhERCZUKj4iIhEqFR0REQvVf3hCRyERjsL4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show = 30\n",
    "plt.scatter(Z[:show,0],Z[:show,1])\n",
    "for i in range(show):\n",
    "    plt.annotate(s=id2w[i],xy=(Z[i,0],Z[i,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}