splitting_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "\n",
    "from data.utils import split_dataset_binary\n",
    "from data.preprocess import concat_unshared_task_datasets, save_preprocessed_data, preprocess_tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unshared task dataset concat done.\n",
      "Label Count: Sexism-3941, Racism-2062, None-12970\n"
     ]
    }
   ],
   "source": [
    "data = concat_unshared_task_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "idx_racism = list(range(len(data[\"racism\"])))\n",
    "idx_sexism = list(range(len(data[\"sexism\"])))\n",
    "idx_none = list(range(len(data[\"none\"])))\n",
    "\n",
    "random.shuffle(idx_racism)\n",
    "random.shuffle(idx_sexism)\n",
    "random.shuffle(idx_none)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def preprocess(tweets):\n",
    "    hashtag = []\n",
    "    no_hashtag = []\n",
    "    for tweet in tweets:\n",
    "        _tweet = preprocess_tweet(tweet, removeHashTag=False)\n",
    "        __tweet = preprocess_tweet(tweet, removeHashTag=True)\n",
    "        if _tweet and __tweet:\n",
    "            hashtag.append(_tweet)\n",
    "            no_hashtag.append(__tweet)\n",
    "    assert len(hashtag) == len(no_hashtag)\n",
    "    return hashtag, no_hashtag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_preprocessed_train_test_set(name, x_neg, x_pos):\n",
    "    x_train, y_train, x_valid, y_valid, x_test, y_test = split_dataset_binary(x_neg=x_neg, x_pos=x_pos, split=[0.8, 0.1, 0.1])\n",
    "    x_test = np.concatenate((x_valid, x_test))\n",
    "    y_test = np.concatenate((y_valid, y_test))\n",
    "    x_train_yes, x_train_no = preprocess(x_train)\n",
    "    x_test_yes, x_test_no = preprocess(x_test)\n",
    "    \n",
    "    save_preprocessed_data(name + \"_yes\", hasValid=False, data_={\"train\": (x_train_yes, y_train), \"test\": (x_test_yes, y_test)})\n",
    "    save_preprocessed_data(name + \"_no\", hasValid=False, data_={\"train\": (x_train_no, y_train), \"test\": (x_test_no, y_test)})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sexism binary test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "split index\n",
      "[2594, 5188, 7782, 10376]\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "concatenating chunks\n",
      "merged and splitted to shape x_train:(10375,), y_train:(10375,), x_test:(2595,), y_test:(2595,)\n",
      "Training set size:13527 (neg:10375/pos:3152)\n",
      "\n",
      "sample neg - Wow. I might be out of blue hair dye. But I have a tube of Pravana that is purple... // @TheQuinnspiracy\n",
      "sample pos - RT @Vickisecret__: These NSW promo girls think way too highly of themselves.. They're not even attractive 😳 #MKR #MKR2015\n",
      "Valid set size:1691 (neg:1297/pos:394)\n",
      "\n",
      "sample neg - @LYH786 Looks much to intelligent to be Baghdadi ;)\n",
      "sample pos - RT @RykerDomz When a woman gets in a wreck I'm not surprised. #notsexist\n",
      "Test set size:1693 (neg:1298/pos:395)\n",
      "\n",
      "sample neg - Can a camper please push #katandandre into that lake? Wash that #smug look of their face #mkr\n",
      "sample pos - How unfortunate that no one cares what you think. @E_B_A_H__3 Im not sexist, but football ain't meant for women\n",
      "Wrote on train_sexism_hashtag_yes.txt\n",
      "Wrote on test_sexism_hashtag_yes.txt\n",
      "Wrote on train_sexism_hashtag_no.txt\n",
      "Wrote on test_sexism_hashtag_no.txt\n"
     ]
    }
   ],
   "source": [
    "save_preprocessed_train_test_set(\"sexism_hashtag\", data[\"none\"], data[\"sexism\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Racism Binary test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "split index\n",
      "[2594, 5188, 7782, 10376]\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "divided into chunk shape _x: (2594,), _y: (2594,)\n",
      "concatenating chunks\n",
      "merged and splitted to shape x_train:(10375,), y_train:(10375,), x_test:(2595,), y_test:(2595,)\n",
      "Training set size:12024 (neg:10375/pos:1649)\n",
      "\n",
      "sample neg - I have faith in the project, but more awareness needs to be spread about the difficulties and harassment that many female FOSS devs face.\n",
      "sample pos - @QrysBinThynkn That's because it's all about the barbarity of Islam, not the beards.\n",
      "Valid set size:1503 (neg:1297/pos:206)\n",
      "\n",
      "sample neg - RT @octal: Wow. This really isn't how to do customer support. https://t.co/fHLfHyunuu\n",
      "sample pos - @Vandaliser @sajid_fairooz @IsraeliRegime Science was moving forward in India and Persia before Islam. Islam only slowed it down.\n",
      "Test set size:1505 (neg:1298/pos:207)\n",
      "\n",
      "sample neg - @Tilaurin probably :P\n",
      "sample pos - @DianH4 But it really doesn't matter who the Mujahedeen animals are fighting. They have been murdering someone for 1400 years.\n",
      "Wrote on train_racism_hashtag_yes.txt\n",
      "Wrote on test_racism_hashtag_yes.txt\n",
      "Wrote on train_racism_hashtag_no.txt\n",
      "Wrote on test_racism_hashtag_no.txt\n"
     ]
    }
   ],
   "source": [
    "save_preprocessed_train_test_set(\"racism_hashtag\", data[\"none\"], data[\"racism\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Loading "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_abusive_final_binary.txt   train_racism_binary.txt\r\n",
      "test_racism_binary.txt          train_racism_final2_binary.txt\r\n",
      "test_racism_final2_binary.txt   train_racism_final_binary.txt\r\n",
      "test_racism_final_binary.txt    train_racism_hashtag_no.txt\r\n",
      "test_racism_hashtag_no.txt      train_racism_hashtag_yes.txt\r\n",
      "test_racism_hashtag_yes.txt     train_sexism_binary.txt\r\n",
      "test_sexism_binary.txt          train_sexism_final2_binary.txt\r\n",
      "test_sexism_final2_binary.txt   train_sexism_final_binary.txt\r\n",
      "test_sexism_final_binary.txt    train_sexism_hashtag_no.txt\r\n",
      "test_sexism_hashtag_no.txt      train_sexism_hashtag_yes.txt\r\n",
      "test_sexism_hashtag_yes.txt     train_sexism_no_ht_binary.txt\r\n",
      "test_sexism_no_ht_binary.txt    valid_racism_binary.txt\r\n",
      "train_abusive_final_binary.txt  valid_sexism_binary.txt\r\n"
     ]
    }
   ],
   "source": [
    "%ls data/preprocessed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from data.preprocess import load_from_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = load_from_file(\"sexism_hashtag_no\")\n",
    "data2 = load_from_file(\"sexism_hashtag_yes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ \"i didn't know you could throw interceptions and make tackles in the kitchen.\",\n",
       "       'define feminazi',\n",
       "       \"i'm not sexist but don't go around saying wemon are top shit and yous run the world. when it's men that literally run …\",\n",
       "       'ha ha suck on that kat and andre you dumb pieces of shit.',\n",
       "       'is bad because i should be able to hate entire demographics of people for the bad things individuals do to me!',\n",
       "       \"if you are a women,and this is no sexist comment directly, and we are watching football just don't talk unless my team scores\",\n",
       "       \"i'm really not sexist, but most women can't drive for shit!!\",\n",
       "       'so...is the answer \"nothing\"?',\n",
       "       \"i support because i'm a gamer and i don't want a bunch of corrupt media feminazi's ruining video games for everyone.\",\n",
       "       \"these nsw promo girls think way too highly of themselves.. they're not even attractive 😳\"], \n",
       "      dtype='<U153')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"x_train\"][-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ \"i didn't know you could throw interceptions and make tackles in the kitchen.\",\n",
       "       'define feminazi',\n",
       "       \"i'm not sexist but don't go around saying wemon are top shit and yous run the world. when it's men that literally run …\",\n",
       "       'ha ha suck on that kat and andre you dumb pieces of shit. mkr dumbslut',\n",
       "       'blameonenotall is bad because i should be able to hate entire demographics of people for the bad things individuals do to me! racismyall',\n",
       "       \"if you are a women,and this is no sexist comment directly, and we are watching football just don't talk unless my team scores\",\n",
       "       \"i'm really not sexist, but most women can't drive for shit!!\",\n",
       "       'so...is the answer \"nothing\"?',\n",
       "       \"i support gamergate because i'm a gamer and i don't want a bunch of corrupt media feminazi's ruining video games for everyone.\",\n",
       "       \"these nsw promo girls think way too highly of themselves.. they're not even attractive 😳 mkr mkr2015\"], \n",
       "      dtype='<U157')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2[\"x_train\"][-10:]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}