Generating the Similarity Matrix.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Similarty matrix \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sc\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.20099636,  0.05064982],\n",
       "       [ 0.99286728,  0.39290391],\n",
       "       [ 0.1397375 ,  0.0186377 ]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.rand(3,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('dengue_neo.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Title</th>\n",
       "      <th>Year</th>\n",
       "      <th>Source.title</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Issue</th>\n",
       "      <th>Art..No.</th>\n",
       "      <th>Page.start</th>\n",
       "      <th>Page.end</th>\n",
       "      <th>...</th>\n",
       "      <th>Conference.code</th>\n",
       "      <th>ISSN</th>\n",
       "      <th>ISBN</th>\n",
       "      <th>CODEN</th>\n",
       "      <th>PubMed.ID</th>\n",
       "      <th>Language.of.Original.Document</th>\n",
       "      <th>Abbreviated.Source.Title</th>\n",
       "      <th>Document.Type</th>\n",
       "      <th>Source</th>\n",
       "      <th>EID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Gouvêa M.M.; Jr.</td>\n",
       "      <td>Time-spatial model on the dynamics of the prol...</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>Communications in Nonlinear Science and Numeri...</td>\n",
       "      <td>44</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>130</td>\n",
       "      <td>143</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10075704</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>English</td>\n",
       "      <td>Comm. Nonlinear Sci. Numer. Simul.</td>\n",
       "      <td>Article</td>\n",
       "      <td>Scopus</td>\n",
       "      <td>2-s2.0-84981507175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>Wang X.; Tang S.; Cheke R.A.</td>\n",
       "      <td>A stage structured mosquito model incorporatin...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>Journal of Theoretical Biology</td>\n",
       "      <td>411</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>225193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JTBIA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>English</td>\n",
       "      <td>J. Theor. Biol.</td>\n",
       "      <td>Article</td>\n",
       "      <td>Scopus</td>\n",
       "      <td>2-s2.0-84991688005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>Tang B.; Xiao Y.; Tang S.; Wu J.</td>\n",
       "      <td>Modelling weekly vector control against Dengue...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>Journal of Theoretical Biology</td>\n",
       "      <td>410</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65</td>\n",
       "      <td>76</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>225193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JTBIA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>English</td>\n",
       "      <td>J. Theor. Biol.</td>\n",
       "      <td>Article</td>\n",
       "      <td>Scopus</td>\n",
       "      <td>2-s2.0-84988473946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>Delmelle E.; Hagenlocher M.; Kienberger S.; Ca...</td>\n",
       "      <td>A spatial model of socioeconomic and environme...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>Acta Tropica</td>\n",
       "      <td>164</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>169</td>\n",
       "      <td>176</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0001706X</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ACTRA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>English</td>\n",
       "      <td>Acta Trop.</td>\n",
       "      <td>Article</td>\n",
       "      <td>Scopus</td>\n",
       "      <td>2-s2.0-84988037437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>Rodrigues H.S.; Monteiro M.T.T.; Torres D.F.M.</td>\n",
       "      <td>Seasonality effects on dengue: basic reproduct...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>Mathematical Methods in the Applied Sciences</td>\n",
       "      <td>39</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4671</td>\n",
       "      <td>4679</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1704214</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MMSCD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>English</td>\n",
       "      <td>Math Methods Appl Sci</td>\n",
       "      <td>Conference Paper</td>\n",
       "      <td>Scopus</td>\n",
       "      <td>2-s2.0-84920399968</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                            Authors  \\\n",
       "0           1                                   Gouvêa M.M.; Jr.   \n",
       "1           3                       Wang X.; Tang S.; Cheke R.A.   \n",
       "2           4                   Tang B.; Xiao Y.; Tang S.; Wu J.   \n",
       "3           5  Delmelle E.; Hagenlocher M.; Kienberger S.; Ca...   \n",
       "4           6     Rodrigues H.S.; Monteiro M.T.T.; Torres D.F.M.   \n",
       "\n",
       "                                               Title    Year  \\\n",
       "0  Time-spatial model on the dynamics of the prol...  2017.0   \n",
       "1  A stage structured mosquito model incorporatin...  2016.0   \n",
       "2  Modelling weekly vector control against Dengue...  2016.0   \n",
       "3  A spatial model of socioeconomic and environme...  2016.0   \n",
       "4  Seasonality effects on dengue: basic reproduct...  2016.0   \n",
       "\n",
       "                                        Source.title Volume Issue Art..No.  \\\n",
       "0  Communications in Nonlinear Science and Numeri...     44   NaN      NaN   \n",
       "1                     Journal of Theoretical Biology    411   NaN      NaN   \n",
       "2                     Journal of Theoretical Biology    410   NaN      NaN   \n",
       "3                                       Acta Tropica    164   NaN      NaN   \n",
       "4       Mathematical Methods in the Applied Sciences     39    16      NaN   \n",
       "\n",
       "  Page.start Page.end         ...          Conference.code      ISSN ISBN  \\\n",
       "0        130      143         ...                      NaN  10075704  NaN   \n",
       "1         27       36         ...                      NaN    225193  NaN   \n",
       "2         65       76         ...                      NaN    225193  NaN   \n",
       "3        169      176         ...                      NaN  0001706X  NaN   \n",
       "4       4671     4679         ...                      NaN   1704214  NaN   \n",
       "\n",
       "   CODEN PubMed.ID Language.of.Original.Document  \\\n",
       "0    NaN       NaN                       English   \n",
       "1  JTBIA       NaN                       English   \n",
       "2  JTBIA       NaN                       English   \n",
       "3  ACTRA       NaN                       English   \n",
       "4  MMSCD       NaN                       English   \n",
       "\n",
       "             Abbreviated.Source.Title     Document.Type  Source  \\\n",
       "0  Comm. Nonlinear Sci. Numer. Simul.           Article  Scopus   \n",
       "1                     J. Theor. Biol.           Article  Scopus   \n",
       "2                     J. Theor. Biol.           Article  Scopus   \n",
       "3                          Acta Trop.           Article  Scopus   \n",
       "4               Math Methods Appl Sci  Conference Paper  Scopus   \n",
       "\n",
       "                  EID  \n",
       "0  2-s2.0-84981507175  \n",
       "1  2-s2.0-84991688005  \n",
       "2  2-s2.0-84988473946  \n",
       "3  2-s2.0-84988037437  \n",
       "4  2-s2.0-84920399968  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Unnamed: 0' 'Authors' 'Title' 'Year' 'Source.title' 'Volume' 'Issue'\n",
      " 'Art..No.' 'Page.start' 'Page.end' 'Page.count' 'Cited.by' 'DOI' 'Link'\n",
      " 'Affiliations' 'Authors.with.affiliations' 'Abstract' 'Author.Keywords'\n",
      " 'Index.Keywords' 'Molecular.Sequence.Numbers' 'Chemicals.CAS' 'Tradenames'\n",
      " 'Manufacturers' 'Funding.Details' 'References' 'Correspondence.Address'\n",
      " 'Editors' 'Sponsors' 'Publisher' 'Conference.name' 'Conference.date'\n",
      " 'Conference.location' 'Conference.code' 'ISSN' 'ISBN' 'CODEN' 'PubMed.ID'\n",
      " 'Language.of.Original.Document' 'Abbreviated.Source.Title' 'Document.Type'\n",
      " 'Source' 'EID']\n"
     ]
    }
   ],
   "source": [
    "print(df.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Some complex physical systems, such as cellular regulation, ecosystems, and societies, can be represented by local interactions between agents. Then, complex behaviors may emerge. A cellular automaton is a discrete dynamic system with these features. Among the several complex systems, epidemic diseases are given special attention by researchers with respect to their dynamics. Understanding the behavior of an epidemic may well benefit a society. For instance, different proliferation scenarios may be produced and a prevention policy set. This paper presents a new simulation method of the time-spatial spread of the Dengue mosquito with a cellular automaton. Thus, it will be possible to create different dissemination scenarios and preventive policies for these in several regions. Simulations were performed with different initial conditions and parameters as a result of which the behavior of the proposed method was characterized. © 2016 Elsevier B.V.'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tokens0 = nltk.wordpunct_tokenize(t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Some',\n",
       " 'complex',\n",
       " 'physical',\n",
       " 'systems',\n",
       " ',',\n",
       " 'such',\n",
       " 'as',\n",
       " 'cellular',\n",
       " 'regulation',\n",
       " ',',\n",
       " 'ecosystems',\n",
       " ',',\n",
       " 'and',\n",
       " 'societies',\n",
       " ',',\n",
       " 'can',\n",
       " 'be',\n",
       " 'represented',\n",
       " 'by',\n",
       " 'local',\n",
       " 'interactions',\n",
       " 'between',\n",
       " 'agents',\n",
       " '.',\n",
       " 'Then',\n",
       " ',',\n",
       " 'complex',\n",
       " 'behaviors',\n",
       " 'may',\n",
       " 'emerge',\n",
       " '.',\n",
       " 'A',\n",
       " 'cellular',\n",
       " 'automaton',\n",
       " 'is',\n",
       " 'a',\n",
       " 'discrete',\n",
       " 'dynamic',\n",
       " 'system',\n",
       " 'with',\n",
       " 'these',\n",
       " 'features',\n",
       " '.',\n",
       " 'Among',\n",
       " 'the',\n",
       " 'several',\n",
       " 'complex',\n",
       " 'systems',\n",
       " ',',\n",
       " 'epidemic',\n",
       " 'diseases',\n",
       " 'are',\n",
       " 'given',\n",
       " 'special',\n",
       " 'attention',\n",
       " 'by',\n",
       " 'researchers',\n",
       " 'with',\n",
       " 'respect',\n",
       " 'to',\n",
       " 'their',\n",
       " 'dynamics',\n",
       " '.',\n",
       " 'Understanding',\n",
       " 'the',\n",
       " 'behavior',\n",
       " 'of',\n",
       " 'an',\n",
       " 'epidemic',\n",
       " 'may',\n",
       " 'well',\n",
       " 'benefit',\n",
       " 'a',\n",
       " 'society',\n",
       " '.',\n",
       " 'For',\n",
       " 'instance',\n",
       " ',',\n",
       " 'different',\n",
       " 'proliferation',\n",
       " 'scenarios',\n",
       " 'may',\n",
       " 'be',\n",
       " 'produced',\n",
       " 'and',\n",
       " 'a',\n",
       " 'prevention',\n",
       " 'policy',\n",
       " 'set',\n",
       " '.',\n",
       " 'This',\n",
       " 'paper',\n",
       " 'presents',\n",
       " 'a',\n",
       " 'new',\n",
       " 'simulation',\n",
       " 'method',\n",
       " 'of',\n",
       " 'the',\n",
       " 'time',\n",
       " '-',\n",
       " 'spatial',\n",
       " 'spread',\n",
       " 'of',\n",
       " 'the',\n",
       " 'Dengue',\n",
       " 'mosquito',\n",
       " 'with',\n",
       " 'a',\n",
       " 'cellular',\n",
       " 'automaton',\n",
       " '.',\n",
       " 'Thus',\n",
       " ',',\n",
       " 'it',\n",
       " 'will',\n",
       " 'be',\n",
       " 'possible',\n",
       " 'to',\n",
       " 'create',\n",
       " 'different',\n",
       " 'dissemination',\n",
       " 'scenarios',\n",
       " 'and',\n",
       " 'preventive',\n",
       " 'policies',\n",
       " 'for',\n",
       " 'these',\n",
       " 'in',\n",
       " 'several',\n",
       " 'regions',\n",
       " '.',\n",
       " 'Simulations',\n",
       " 'were',\n",
       " 'performed',\n",
       " 'with',\n",
       " 'different',\n",
       " 'initial',\n",
       " 'conditions',\n",
       " 'and',\n",
       " 'parameters',\n",
       " 'as',\n",
       " 'a',\n",
       " 'result',\n",
       " 'of',\n",
       " 'which',\n",
       " 'the',\n",
       " 'behavior',\n",
       " 'of',\n",
       " 'the',\n",
       " 'proposed',\n",
       " 'method',\n",
       " 'was',\n",
       " 'characterized',\n",
       " '.',\n",
       " '©',\n",
       " '2016',\n",
       " 'Elsevier',\n",
       " 'B',\n",
       " '.',\n",
       " 'V',\n",
       " '.']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nltk_to = nltk.Text(t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = [w.lower() for w in nltk_to]\n",
    "vocab = sorted(set(words))\n",
    "type(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "filtered_words_T0 = [w for w in tokens0 if not w in stopwords.words('english')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Some', 'complex', 'physical', 'systems', ',', 'cellular', 'regulation', ',', 'ecosystems', ',', 'societies', ',', 'represented', 'local', 'interactions', 'agents', '.', 'Then', ',', 'complex', 'behaviors', 'may', 'emerge', '.', 'A', 'cellular', 'automaton', 'discrete', 'dynamic', 'system', 'features', '.', 'Among', 'several', 'complex', 'systems', ',', 'epidemic', 'diseases', 'given', 'special', 'attention', 'researchers', 'respect', 'dynamics', '.', 'Understanding', 'behavior', 'epidemic', 'may', 'well', 'benefit', 'society', '.', 'For', 'instance', ',', 'different', 'proliferation', 'scenarios', 'may', 'produced', 'prevention', 'policy', 'set', '.', 'This', 'paper', 'presents', 'new', 'simulation', 'method', 'time', '-', 'spatial', 'spread', 'Dengue', 'mosquito', 'cellular', 'automaton', '.', 'Thus', ',', 'possible', 'create', 'different', 'dissemination', 'scenarios', 'preventive', 'policies', 'several', 'regions', '.', 'Simulations', 'performed', 'different', 'initial', 'conditions', 'parameters', 'result', 'behavior', 'proposed', 'method', 'characterized', '.', '©', '2016', 'Elsevier', 'B', '.', 'V', '.']\n"
     ]
    }
   ],
   "source": [
    "print(filtered_words_T0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['some', 'complex', 'physical', 'systems', 'cellular', 'regulation', 'ecosystems', 'societies', 'represented', 'local', 'interactions', 'agents', 'then', 'complex', 'behaviors', 'may', 'emerge', 'a', 'cellular', 'automaton', 'discrete', 'dynamic', 'system', 'features', 'among', 'several', 'complex', 'systems', 'epidemic', 'diseases', 'given', 'special', 'attention', 'researchers', 'respect', 'dynamics', 'understanding', 'behavior', 'epidemic', 'may', 'well', 'benefit', 'society', 'for', 'instance', 'different', 'proliferation', 'scenarios', 'may', 'produced', 'prevention', 'policy', 'set', 'this', 'paper', 'presents', 'new', 'simulation', 'method', 'time', 'spatial', 'spread', 'dengue', 'mosquito', 'cellular', 'automaton', 'thus', 'possible', 'create', 'different', 'dissemination', 'scenarios', 'preventive', 'policies', 'several', 'regions', 'simulations', 'performed', 'different', 'initial', 'conditions', 'parameters', 'result', 'behavior', 'proposed', 'method', 'characterized', '2016', 'elsevier', 'b', 'v']\n"
     ]
    }
   ],
   "source": [
    "filtered_words_t0 = [w.lower() for w in filtered_words_T0 if w.isalnum()]\n",
    "print(filtered_words_t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<91x73 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 88 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vect = CountVectorizer()\n",
    "X_train_counts = count_vect.fit_transform(filtered_words_t0)\n",
    "X_train_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(91, 73)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts)\n",
    "X_train_tf = tf_transformer.transform(X_train_counts)\n",
    "X_train_tf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(91, 73)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_transformer = TfidfTransformer()\n",
    "X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)\n",
    "X_train_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cosine scores ==>  [[ 1.          0.12911647  0.09299782 ...,  0.10117668  0.16888597\n",
      "   0.04722414]\n",
      " [ 0.12911647  1.          0.22658251 ...,  0.13396575  0.17346689\n",
      "   0.05257748]\n",
      " [ 0.09299782  0.22658251  1.         ...,  0.08378385  0.15058275\n",
      "   0.05015608]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "train_set = df['Abstract'].tolist()\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer()\n",
    "tfidf_matrix_train = tfidf_vectorizer.fit_transform(train_set)  #finds the tfidf score with normalization\n",
    "print (\"cosine scores ==> \",cosine_similarity(tfidf_matrix_train, tfidf_matrix_train)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rtn = cosine_similarity(tfidf_matrix_train, tfidf_matrix_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.          0.12911647  0.09299782 ...,  0.10117668  0.16888597\n",
      "  0.04722414]\n"
     ]
    }
   ],
   "source": [
    "print(rtn[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savetxt('docsim10.csv', rtn[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}