men_simlex_disagreement.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from common_imports import *\n",
    "from intrinsic_eval import _men, _ws353, _simlex999\n",
    "\n",
    "sns.timeseries.algo.bootstrap = my_bootstrap\n",
    "sns.categorical.bootstrap = my_bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>w1</th>\n",
       "      <th>w2</th>\n",
       "      <th>sim</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>old</td>\n",
       "      <td>new</td>\n",
       "      <td>1.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>smart</td>\n",
       "      <td>intelligent</td>\n",
       "      <td>9.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hard</td>\n",
       "      <td>difficult</td>\n",
       "      <td>8.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy</td>\n",
       "      <td>cheerful</td>\n",
       "      <td>9.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hard</td>\n",
       "      <td>easy</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      w1           w2   sim\n",
       "0    old          new  1.58\n",
       "1  smart  intelligent  9.20\n",
       "2   hard    difficult  8.77\n",
       "3  happy     cheerful  9.55\n",
       "4   hard         easy  0.95"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simlex.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mmb28/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:2: FutureWarning: convert_objects is deprecated.  Use the data-type specific converters pd.to_datetime, pd.to_timedelta and pd.to_numeric.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "ws, men, simlex = _ws353(), _men(), _simlex999()\n",
    "simlex = simlex.convert_objects(convert_numeric=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>w1</th>\n",
       "      <th>w2</th>\n",
       "      <th>sim</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sun</td>\n",
       "      <td>sunlight</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>automobile</td>\n",
       "      <td>car</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>river</td>\n",
       "      <td>water</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>stair</td>\n",
       "      <td>staircase</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>morning</td>\n",
       "      <td>sunrise</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           w1         w2   sim\n",
       "0         sun   sunlight  50.0\n",
       "1  automobile        car  50.0\n",
       "2       river      water  49.0\n",
       "3       stair  staircase  49.0\n",
       "4     morning    sunrise  49.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "men.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>w1</th>\n",
       "      <th>w2</th>\n",
       "      <th>sim</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>love</td>\n",
       "      <td>sex</td>\n",
       "      <td>6.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tiger</td>\n",
       "      <td>cat</td>\n",
       "      <td>7.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tiger</td>\n",
       "      <td>tiger</td>\n",
       "      <td>10.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>book</td>\n",
       "      <td>paper</td>\n",
       "      <td>7.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>computer</td>\n",
       "      <td>keyboard</td>\n",
       "      <td>7.62</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         w1        w2    sim\n",
       "0      love       sex   6.77\n",
       "1     tiger       cat   7.35\n",
       "2     tiger     tiger  10.00\n",
       "3      book     paper   7.46\n",
       "4  computer  keyboard   7.62"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ws.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>w1</th>\n",
       "      <th>w2</th>\n",
       "      <th>sim</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>old</td>\n",
       "      <td>new</td>\n",
       "      <td>1.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>smart</td>\n",
       "      <td>intelligent</td>\n",
       "      <td>9.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hard</td>\n",
       "      <td>difficult</td>\n",
       "      <td>8.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy</td>\n",
       "      <td>cheerful</td>\n",
       "      <td>9.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hard</td>\n",
       "      <td>easy</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      w1           w2   sim\n",
       "0    old          new  1.58\n",
       "1  smart  intelligent  9.20\n",
       "2   hard    difficult  8.77\n",
       "3  happy     cheerful  9.55\n",
       "4   hard         easy  0.95"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simlex.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ws_pairs = set(tuple(sorted([x.w1, x.w2])) for x in ws.itertuples())\n",
    "men_pairs = set(tuple(sorted([x.w1, x.w2])) for x in men.itertuples())\n",
    "simlex_pairs = set(tuple(sorted([x.w1, x.w2])) for x in simlex.itertuples())\n",
    "\n",
    "ws_dict = {tuple(sorted([x.w1, x.w2])):x.sim for x in ws.itertuples()}\n",
    "men_dict = {tuple(sorted([x.w1, x.w2])):x.sim for x in men.itertuples()}\n",
    "simlex_dict = {tuple(sorted([x.w1, x.w2])):x.sim for x in simlex.itertuples()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('bacon', 'meat') 44.0 5.8 0.88 0.58 0.3\n",
      "('day', 'morning') 41.0 4.87 0.82 0.487 0.333\n",
      "('bathroom', 'kitchen') 43.0 3.1 0.86 0.31 0.55\n",
      "('cattle', 'sheep') 38.0 4.77 0.76 0.477 0.283\n",
      "('breakfast', 'dinner') 37.0 3.33 0.74 0.333 0.407\n",
      "('sunrise', 'sunset') 42.0 2.47 0.84 0.247 0.593\n",
      "('lamb', 'sheep') 40.0 8.42 0.8 0.842 -0.042\n",
      "('day', 'night') 39.0 1.88 0.78 0.188 0.592\n",
      "('dawn', 'night') 36.0 2.95 0.72 0.295 0.425\n",
      "('bed', 'room') 37.0 2.35 0.74 0.235 0.505\n",
      "('door', 'kitchen') 24.0 1.67 0.48 0.167 0.313\n",
      "('bird', 'hawk') 44.0 7.85 0.88 0.785 0.095\n",
      "('lunch', 'sandwich') 42.0 6.3 0.84 0.63 0.21\n",
      "('door', 'floor') 28.0 1.67 0.56 0.167 0.393\n",
      "('beach', 'sea') 44.0 4.68 0.88 0.468 0.412\n",
      "('river', 'valley') 37.0 1.67 0.74 0.167 0.573\n",
      "('snow', 'weather') 37.0 5.48 0.74 0.548 0.192\n",
      "('bath', 'room') 35.0 3.33 0.7 0.333 0.367\n",
      "('camera', 'lens') 43.0 4.28 0.86 0.428 0.432\n",
      "('floor', 'roof') 39.0 2.62 0.78 0.262 0.518\n",
      "('bathroom', 'bedroom') 38.0 3.4 0.76 0.34 0.42\n",
      "('fun', 'game') 35.0 3.42 0.7 0.342 0.358\n",
      "('mist', 'rain') 41.0 5.97 0.82 0.597 0.223\n",
      "('door', 'window') 36.0 3.33 0.72 0.333 0.387\n",
      "('beef', 'cattle') 41.0 7.03 0.82 0.703 0.117\n",
      "('summer', 'winter') 37.0 2.38 0.74 0.238 0.502\n",
      "('friend', 'guy') 26.0 3.88 0.52 0.388 0.132\n",
      "('bed', 'bedroom') 42.0 3.4 0.84 0.34 0.5\n",
      "('ceiling', 'roof') 39.0 7.58 0.78 0.758 0.022\n",
      "('clothes', 'jacket') 43.0 5.15 0.86 0.515 0.345\n",
      "('cloud', 'mist') 37.0 6.67 0.74 0.667 0.073\n",
      "('maple', 'oak') 43.0 6.03 0.86 0.603 0.257\n",
      "('ocean', 'sea') 46.0 8.27 0.92 0.827 0.093\n",
      "('chicken', 'rice') 34.0 1.43 0.68 0.143 0.537\n",
      "('flower', 'violet') 34.0 6.95 0.68 0.695 -0.015\n",
      "('man', 'woman') 42.0 3.33 0.84 0.333 0.507\n",
      "('alley', 'street') 37.0 5.48 0.74 0.548 0.192\n",
      "('ceiling', 'floor') 38.0 1.73 0.76 0.173 0.587\n",
      "('canyon', 'valley') 36.0 6.75 0.72 0.675 0.045\n",
      "('sky', 'sun') 40.0 2.27 0.8 0.227 0.573\n",
      "('bed', 'furniture') 45.0 6.08 0.9 0.608 0.292\n",
      "('coast', 'shore') 40.0 8.83 0.8 0.883 -0.083\n",
      "('eye', 'tear') 41.0 3.55 0.82 0.355 0.465\n",
      "('brick', 'wall') 37.0 4.68 0.74 0.468 0.272\n",
      "('river', 'stream') 45.0 7.3 0.9 0.73 0.17\n"
     ]
    }
   ],
   "source": [
    "from itertools import product\n",
    "for pair in product(men_pairs.intersection(simlex_pairs)):\n",
    "    score1, score2 = men_dict[pair[0]]/50, simlex_dict[pair[0]]/10\n",
    "    print(pair[0], men_dict[pair[0]], simlex_dict[pair[0]], score1, score2, score1-score2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('psychology', 'science') 0.179\n",
      "('moon', 'planet') 0.221\n",
      "('coast', 'shore') 0.027\n",
      "('dawn', 'day') 0.206\n",
      "('professor', 'student') 0.486\n",
      "('doctor', 'professor') 0.197\n",
      "('aluminum', 'metal') 0.058\n",
      "('closet', 'clothes') 0.473\n",
      "('man', 'woman') 0.497\n"
     ]
    }
   ],
   "source": [
    "for pair in product(ws_pairs.intersection(simlex_pairs)):\n",
    "    print(pair[0], abs(ws_dict[pair[0]]/10 - simlex_dict[pair[0]]/10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('coast', 'hill') 0.122\n",
      "('automobile', 'car') 0.106\n",
      "('baby', 'mother') 0.055\n",
      "('coast', 'shore') 0.11\n",
      "('food', 'fruit') 0.132\n",
      "('man', 'woman') 0.01\n"
     ]
    }
   ],
   "source": [
    "for pair in product(ws_pairs.intersection(men_pairs)):\n",
    "    print(pair[0], abs(ws_dict[pair[0]]/10 - men_dict[pair[0]]/50))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}