preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Packages & Global Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from LIWC.liwc_func import Liwc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "RAW_DIR    = \"data/raw_data\"\n",
    "OUTPUT_DIR = \"data/proc_data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isdir(OUTPUT_DIR):\n",
    "    os.makedirs(OUTPUT_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "LIWC = Liwc(\"en\")\n",
    "\n",
    "LIWC_FEATURE_LIST = ['words_per_sentence', 'six_plus_words', 'word_count', 'function', 'pronoun', 'ppron', 'i', 'we', 'you', 'she', 'he', 'they', 'ipron', 'article', 'prep', 'auxverb', 'adverb', 'conj', 'negate', 'verb', 'adj', 'compare', 'interrog', 'number', 'quant', 'affect', 'posemo', 'negemo', 'anx', 'anger', 'sad', 'social', 'family', 'friend', 'female', 'male', 'cogproc', 'insight', 'cause', 'discrep', 'tentat', 'certain', 'differ', 'percept', 'see', 'hear', 'feel', 'bio', 'body', 'health', 'sexual', 'ingest', 'drives', 'affiliation', 'achieve', 'power', 'reward', 'risk', 'focuspast', 'focuspresent', 'focusfuture', 'relativ', 'motion', 'space', 'time', 'work', 'leisure', 'home', 'money', 'relig', 'death', 'informal', 'swear', 'netspeak', 'assent', 'nonflu', 'filler']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"label\": [],\n",
    "    \"text\": [],\n",
    "}\n",
    "data = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "for split in [\"train\", \"validation\", \"test\"]:\n",
    "    df = pd.read_csv(os.path.join(RAW_DIR, split + \".tsv\"), delimiter=\"\\t\", header=None)\n",
    "    df.drop([0, 3, 4, 8, 9, 10, 11, 12], axis=1, inplace=True)\n",
    "    df.rename(columns={1: \"label\", 2: \"text\", 5: \"job\", 6: \"state\", 7: \"party\", 13: \"context\"}, inplace=True)\n",
    "    df = df[df[\"label\"] != \"half-true\"]\n",
    "    df.loc[(df[\"label\"] == \"true\") | (df[\"label\"] == \"mostly-true\"), \"label\"] = 1\n",
    "    df.loc[(df[\"label\"] == \"false\") | (df[\"label\"] == \"barely-true\") | (df[\"label\"] == \"pants-fire\"), \"label\"] = 0\n",
    "    data = pd.concat([data, df], axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Sensitive Attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"democrat\"] = data[\"party\"].apply(lambda x: 1 if x == \"democrat\" else 0)\n",
    "data[\"republican\"] = data[\"party\"].apply(lambda x: 1 if x == \"republican\" else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Categorical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),\n",
    "    ('encoder', OneHotEncoder(handle_unknown='ignore', max_categories=20))])\n",
    "\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('cat', categorical_transformer, [0])\n",
    "    ])\n",
    "    \n",
    "def one_hot_encode(df, column_name):\n",
    "    col = preprocessor.fit_transform(df[column_name].values.reshape(-1, 1)).toarray()\n",
    "    df = df.drop(column_name, axis=1)\n",
    "    for i in range(col.shape[1]):\n",
    "        df[column_name + \"_\" + str(i)] = col[:, i]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = one_hot_encode(data, \"job\")\n",
    "data = one_hot_encode(data, \"state\")\n",
    "data = one_hot_encode(data, \"party\")\n",
    "data = one_hot_encode(data, \"context\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Textual Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "liwc_features = {k: [] for k in LIWC_FEATURE_LIST}\n",
    "for i in range(len(data)):\n",
    "    LIWC_result = LIWC.cal_liwc(data.iloc[i][\"text\"])\n",
    "    for f in LIWC_FEATURE_LIST:\n",
    "        liwc_features[f].append(LIWC_result[f] if f in LIWC_result else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in LIWC_FEATURE_LIST:\n",
    "    data[f] = liwc_features[f]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.to_pickle(data, os.path.join(OUTPUT_DIR, \"data.pkl\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}