Extract_Z_from_Screening.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "import scipy, requests, codecs, os, re, nltk, itertools, csv\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import tensorflow as tf\n",
    "from scipy.stats import spearmanr\n",
    "import pandas as pd\n",
    "import functools as ft\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import normalize\n",
    "import gdown\n",
    "import random\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load GloVe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "322636"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def loadWordVecs(model_str):\n",
    "    word_dictionary = {}\n",
    "    \n",
    "    input_file_destination = model_str +'_wiki_vectors.txt'\n",
    "\n",
    "    f = codecs.open(input_file_destination, 'r', 'utf-8') \n",
    "    x = 0\n",
    "    \n",
    "    count = 0\n",
    "    for line in f:\n",
    "        count +=1\n",
    "        line = line.split(\" \", 1)\n",
    "        if len(line) != 2:\n",
    "            print(count)\n",
    "            continue\n",
    "        transformed_key = line[0]\n",
    "\n",
    "        try:\n",
    "            transformed_key = str(transformed_key)\n",
    "\n",
    "        except:\n",
    "            print(\"Can't convert the key to unicode:\", transformed_key)\n",
    "\n",
    "        word_dictionary[transformed_key] = np.fromstring(line[1], dtype=\"float32\", sep=\" \")\n",
    "\n",
    "        if word_dictionary[transformed_key].shape[0] != 300 and x == 0:\n",
    "            print(transformed_key, word_dictionary[transformed_key].shape)\n",
    "            x += 1\n",
    "\n",
    "    return  word_dictionary     \n",
    "\n",
    "orig_glove = loadWordVecs('glove')\n",
    "len(orig_glove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select set Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_word_of_desired_pos(word, pos):\n",
    "    words = []\n",
    "    for w in word:\n",
    "        # no single letter word\n",
    "        if len(w) == 1:\n",
    "            continue\n",
    "        tag = nltk.pos_tag([w])\n",
    "        if tag[0][1] in pos:\n",
    "            words.append(tag[0][0])\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "desired_pos = [\n",
    "                'JJ', # adjective base form -> comparative + superlative  + adverb\n",
    "                # 'NN', # singular noun -> plural 'NNS'\n",
    "                # 'NNP', # proper noun singular 'Harrison' -> proper noun plural 'NNPS'\n",
    "                # 'PRP', # personal pronoun 'he' -> possesive pronoun 'his' PRP$' \n",
    "                # 'VB', # verb base form 'take' -> 'VBD' took, 'VBG' taking, 'VBN' taken, 'VBZ' takes\n",
    "                # 'VBP', # single present, non-3rd person 'take'\n",
    "                ]\n",
    "noun_pos = ['NN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gender_dist(word, gender_mat):\n",
    "    score = []\n",
    "    word = np.array(word)\n",
    "    word = word/norm(word)\n",
    "    score = word.dot(gender_mat)\n",
    "    return np.mean(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_word_of_pos_verbose(all_words, desired_pos):\n",
    "    start = 0\n",
    "    selected_words = []\n",
    "    while start < len(all_words):\n",
    "        if start>10000:\n",
    "            print('Scanning words %d/%d'%(start/10000, int(len(all_words)/10000)))\n",
    "        end = start + 10000\n",
    "        if end > len(all_words):\n",
    "            end = len(all_words)\n",
    "        selected_words = selected_words + select_word_of_desired_pos(all_words[start:end], desired_pos)\n",
    "        start = start + 10000\n",
    "    return selected_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load gendered words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "female_words = pd.read_csv('./data/A_female.csv')\n",
    "female_words = list(female_words['0'])\n",
    "\n",
    "male_words = pd.read_csv('./data/A_male.csv')\n",
    "male_words = list(male_words['0'])\n",
    "\n",
    "gender_words = male_words+female_words\n",
    "\n",
    "gender_mat = np.array([orig_glove[w] for w in gender_words])\n",
    "norm_gender_mat = gender_mat/norm(gender_mat, axis=1).reshape(-1,1)\n",
    "norm_gender_mat = norm_gender_mat.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Construct set Z and save the rest as set X."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing file ./screening/X90_names.txt\n"
     ]
    }
   ],
   "source": [
    "f = open('./screening/output.txt', 'w', encoding='utf-8')\n",
    "f.close()\n",
    "\n",
    "s = '90'\n",
    "# load words\n",
    "from_file = './screening/X'+s+'_names.txt'\n",
    "\n",
    "all_words=[]\n",
    "with open(from_file, \"r+\", encoding='utf-8') as f_in:\n",
    "    for line in f_in:\n",
    "        all_words.append(line.replace('\\n',''))   \n",
    "print('Processing file '+ from_file)\n",
    "\n",
    "# delete corrupted ones\n",
    "to_del = []\n",
    "for w in all_words:\n",
    "    if w not in orig_glove.keys():\n",
    "        to_del.append(w)\n",
    "for w in to_del:\n",
    "    all_words.remove(w)\n",
    "\n",
    "# filter out adj and nouns\n",
    "adj_words = select_word_of_pos_verbose(all_words, desired_pos)\n",
    "noun_words = all_words\n",
    "\n",
    "# compute cosine similarity\n",
    "word_dist = []\n",
    "for word in noun_words:\n",
    "    word_dist.append(gender_dist(orig_glove[word], norm_gender_mat))\n",
    "    if len(word_dist)%20000 == 0:\n",
    "        print(datetime.datetime.now(), \"  \", len(word_dist), ' / ', len(word_df.columns))\n",
    "word_dist = np.array(word_dist)\n",
    "std = word_dist.std()\n",
    "\n",
    "\n",
    "extracted_words = [all_words[1]]\n",
    "extracted_mat = np.array([orig_glove[w] for w in extracted_words]).T\n",
    "if len(extracted_words) > 0:\n",
    "    extracted_df = pd.DataFrame(extracted_mat, columns=extracted_words)\n",
    "else:\n",
    "    extracted_df = dummy_df\n",
    "with open('./screening/E_mat_'+s+'.csv', 'w', encoding='utf-8') as file:\n",
    "    extracted_df.to_csv(file, index=True, header=True, line_terminator='\\n')\n",
    "\n",
    "# save the rest to X\n",
    "words_rest = set(all_words) - set(extracted_words)\n",
    "mat_rest = np.array([orig_glove[w] for w in words_rest]).T\n",
    "df_rest = pd.DataFrame(mat_rest, columns=words_rest)\n",
    "with open('./screening/X_mat_'+s+'.csv', 'w', encoding='utf-8') as file:\n",
    "    df_rest.to_csv(file, index=True, header=True, line_terminator='\\n')\n",
    "\n",
    "with open('./screening/output.txt', 'a', encoding='utf-8') as file:\n",
    "    file.write('Length of X_mat_'+s+'.csv is '+str(len(extracted_df.columns))+' out of '+str(len(all_words))\n",
    "               +', accounting to '+str(round(len(extracted_df.columns)/len(all_words)*100))+'%.\\n')\n",
    "with open('./screening/output.txt', 'a', encoding='utf-8') as file:\n",
    "    file.write('\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}