word_vecs_dataframe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''Creating a data structure that is easier to work with than the entire word2vec model from google\n",
    "We identify unique words - stop_words from the Clinton documents, get their word vectors from google's model and save\n",
    "those vectors in a data_frame\n",
    "\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "from os import listdir\n",
    "import numpy as np\n",
    "from pyemd import emd\n",
    "import re\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "from scipy.spatial.distance import euclidean as euc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DIR = \"keywordsExtRAKE/HillarySpeeches/\"\n",
    "\n",
    "#read in file names as list of labels.\n",
    "docLabels = []\n",
    "docLabels = [f for f in listdir(DIR) if f.endswith('.txt')]\n",
    "\n",
    "#create an array of the files we wish to train on.\n",
    "data = []\n",
    "for doc in docLabels:\n",
    "    with open(DIR + doc, 'r') as d:\n",
    "        text = d.read()\n",
    "        text = re.sub(\"[^a-z'.A-Z]\",\" \", text)\n",
    "        data.append(text.lower())\n",
    "\n",
    "# create a list of list of strings, each sublist is a sentence from data, each string is a word in the sentence.\n",
    "sentences = []\n",
    "for i in range(len(data)):\n",
    "    sent = data[i].split('.')\n",
    "    for j in range(len(sent)):\n",
    "        sentences.append(sent[j].split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load googles word2vec model\n",
    "wv = Word2Vec.load_word2vec_format('data/GoogleNews-vectors-negative300.bin.gz', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mystop =[]\n",
    "with open('SmartStoplist.txt','r') as f:\n",
    "    for line in f:\n",
    "        mystop.append(line.strip())\n",
    "\n",
    "#create a list of key words for WMD to consider\n",
    "words = []\n",
    "j = 0\n",
    "for s in sentences:\n",
    "    for i in range(len(s)):\n",
    "        if s[i] not in mystop and s[i] not in words:\n",
    "            words.append(s[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#create a numpy array, each row represents a word vector, if a word is not present in google's model, create a row of zeros\n",
    "vocab_dict = {}\n",
    "for w in words:\n",
    "    try:\n",
    "        vocab_dict[w] = wv[w]\n",
    "    except:\n",
    "        print(\"{} is not in the model\".format{w})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(vocab_dict)\n",
    "file_name = \"word_vecs.csv\"\n",
    "df.to_csv(file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Use CountVectorizer to get vectors of two speeches\n",
    "d1 = data[0]\n",
    "d2 = data[1]\n",
    "vect = CountVectorizer(stop_words=\"english\").fit([d1, d2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# creating a distance matrix for use with these speeches\n",
    "dis = np.zeros((462,462) )\n",
    "for i in range(462):\n",
    "...     for j in range(i+1,462):\n",
    "...         if vect.get_feature_names()[i] in df.columns and vect.get_feature_names()[j] in df.columns:\n",
    "...             dis[i,j] = euc(df[vect.get_feature_names()[i]], df[vect.get_feature_names()[j]])\n",
    "dis = dis + dis.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare the document vecs to work with pyemd\n",
    "v_1, v_2 = vect.transform([d1, d2])\n",
    "v_1 = v_1.toarray().ravel()\n",
    "v_2 = v_2.toarray().ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pyemd needs double precision input\n",
    "v_1 = v_1.astype(np.double)\n",
    "v_2 = v_2.astype(np.double)\n",
    "v_1 /= v_1.sum()\n",
    "v_2 /= v_2.sum()\n",
    "dis = dis.astype(np.double)\n",
    "dis /= dis.max()\n",
    "\n",
    "print(\"d(doc_1, doc_2) = {:.2f}\".format(emd(v_1, v_2, dis)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}