examples/GPA_tutorial/gpa_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a21c86b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2f4a94d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Physics</th>\n",
       "      <th>Biology</th>\n",
       "      <th>History</th>\n",
       "      <th>Second_Language</th>\n",
       "      <th>Geography</th>\n",
       "      <th>Literature</th>\n",
       "      <th>Portuguese_and_Essay</th>\n",
       "      <th>Math</th>\n",
       "      <th>Chemistry</th>\n",
       "      <th>GPA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>622.60</td>\n",
       "      <td>491.56</td>\n",
       "      <td>439.93</td>\n",
       "      <td>707.64</td>\n",
       "      <td>663.65</td>\n",
       "      <td>557.09</td>\n",
       "      <td>711.37</td>\n",
       "      <td>731.31</td>\n",
       "      <td>509.80</td>\n",
       "      <td>1.33333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>538.00</td>\n",
       "      <td>490.58</td>\n",
       "      <td>406.59</td>\n",
       "      <td>529.05</td>\n",
       "      <td>532.28</td>\n",
       "      <td>447.23</td>\n",
       "      <td>527.58</td>\n",
       "      <td>379.14</td>\n",
       "      <td>488.64</td>\n",
       "      <td>2.98333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>455.18</td>\n",
       "      <td>440.00</td>\n",
       "      <td>570.86</td>\n",
       "      <td>417.54</td>\n",
       "      <td>453.53</td>\n",
       "      <td>425.87</td>\n",
       "      <td>475.63</td>\n",
       "      <td>476.11</td>\n",
       "      <td>407.15</td>\n",
       "      <td>1.97333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>756.91</td>\n",
       "      <td>679.62</td>\n",
       "      <td>531.28</td>\n",
       "      <td>583.63</td>\n",
       "      <td>534.42</td>\n",
       "      <td>521.40</td>\n",
       "      <td>592.41</td>\n",
       "      <td>783.76</td>\n",
       "      <td>588.26</td>\n",
       "      <td>2.53333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>584.54</td>\n",
       "      <td>649.84</td>\n",
       "      <td>637.43</td>\n",
       "      <td>609.06</td>\n",
       "      <td>670.46</td>\n",
       "      <td>515.38</td>\n",
       "      <td>572.52</td>\n",
       "      <td>581.25</td>\n",
       "      <td>529.04</td>\n",
       "      <td>1.58667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43298</th>\n",
       "      <td>1</td>\n",
       "      <td>519.55</td>\n",
       "      <td>622.20</td>\n",
       "      <td>660.90</td>\n",
       "      <td>543.48</td>\n",
       "      <td>643.05</td>\n",
       "      <td>579.90</td>\n",
       "      <td>584.80</td>\n",
       "      <td>581.25</td>\n",
       "      <td>573.92</td>\n",
       "      <td>2.76333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43299</th>\n",
       "      <td>1</td>\n",
       "      <td>816.39</td>\n",
       "      <td>851.95</td>\n",
       "      <td>732.39</td>\n",
       "      <td>621.63</td>\n",
       "      <td>810.68</td>\n",
       "      <td>666.79</td>\n",
       "      <td>705.22</td>\n",
       "      <td>781.01</td>\n",
       "      <td>831.76</td>\n",
       "      <td>3.81667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43300</th>\n",
       "      <td>0</td>\n",
       "      <td>798.75</td>\n",
       "      <td>817.58</td>\n",
       "      <td>731.98</td>\n",
       "      <td>648.42</td>\n",
       "      <td>751.30</td>\n",
       "      <td>648.67</td>\n",
       "      <td>662.05</td>\n",
       "      <td>773.15</td>\n",
       "      <td>835.25</td>\n",
       "      <td>3.75000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43301</th>\n",
       "      <td>0</td>\n",
       "      <td>527.66</td>\n",
       "      <td>443.82</td>\n",
       "      <td>545.88</td>\n",
       "      <td>624.18</td>\n",
       "      <td>420.25</td>\n",
       "      <td>676.80</td>\n",
       "      <td>583.41</td>\n",
       "      <td>395.46</td>\n",
       "      <td>509.80</td>\n",
       "      <td>2.50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43302</th>\n",
       "      <td>0</td>\n",
       "      <td>512.56</td>\n",
       "      <td>415.41</td>\n",
       "      <td>517.36</td>\n",
       "      <td>532.37</td>\n",
       "      <td>592.30</td>\n",
       "      <td>382.20</td>\n",
       "      <td>538.35</td>\n",
       "      <td>448.02</td>\n",
       "      <td>496.39</td>\n",
       "      <td>3.16667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43303 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Gender  Physics  Biology  History  Second_Language  Geography  \\\n",
       "0           0   622.60   491.56   439.93           707.64     663.65   \n",
       "1           1   538.00   490.58   406.59           529.05     532.28   \n",
       "2           1   455.18   440.00   570.86           417.54     453.53   \n",
       "3           0   756.91   679.62   531.28           583.63     534.42   \n",
       "4           1   584.54   649.84   637.43           609.06     670.46   \n",
       "...       ...      ...      ...      ...              ...        ...   \n",
       "43298       1   519.55   622.20   660.90           543.48     643.05   \n",
       "43299       1   816.39   851.95   732.39           621.63     810.68   \n",
       "43300       0   798.75   817.58   731.98           648.42     751.30   \n",
       "43301       0   527.66   443.82   545.88           624.18     420.25   \n",
       "43302       0   512.56   415.41   517.36           532.37     592.30   \n",
       "\n",
       "       Literature  Portuguese_and_Essay    Math  Chemistry      GPA  \n",
       "0          557.09                711.37  731.31     509.80  1.33333  \n",
       "1          447.23                527.58  379.14     488.64  2.98333  \n",
       "2          425.87                475.63  476.11     407.15  1.97333  \n",
       "3          521.40                592.41  783.76     588.26  2.53333  \n",
       "4          515.38                572.52  581.25     529.04  1.58667  \n",
       "...           ...                   ...     ...        ...      ...  \n",
       "43298      579.90                584.80  581.25     573.92  2.76333  \n",
       "43299      666.79                705.22  781.01     831.76  3.81667  \n",
       "43300      648.67                662.05  773.15     835.25  3.75000  \n",
       "43301      676.80                583.41  395.46     509.80  2.50000  \n",
       "43302      382.20                538.35  448.02     496.39  3.16667  \n",
       "\n",
       "[43303 rows x 11 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = \"./gpa_data_download/data.csv\"\n",
    "columns_raw = [\"Gender\", \"Physics\", \"Biology\", \n",
    "    \"History\", \"Second_Language\", \"Geography\", \n",
    "    \"Literature\", \"Portuguese_and_Essay\", \n",
    "    \"Math\", \"Chemistry\", \"GPA\"]\n",
    "df = pd.read_csv(f,header=None,names=columns_raw)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3bf6d5cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_seldonian_dataset(input_path,output_path_data,output_path_metadata,columns_raw):\n",
    "    \n",
    "    df = pd.read_csv(input_path,header=None,names=columns_raw)\n",
    "    \n",
    "    # extract features\n",
    "    features = df.iloc[:,1:-1]\n",
    "    \n",
    "    # scale features using standard scaler\n",
    "    scaler = StandardScaler()\n",
    "    scaler.fit(features)\n",
    "    transformed_features = scaler.transform(features)\n",
    "    \n",
    "    # replace features in original dataframe with transformed features\n",
    "    df.iloc[:,1:-1] = transformed_features\n",
    "    \n",
    "    # convert continuous GPA labels to binary: 0 if GPA < 3, 1 if GPA >= 3\n",
    "    # Make a new column called GPA_class\n",
    "    df.loc[:,'GPA_class'] = df['GPA'].apply(lambda x: float(int(x>=3.0)))\n",
    "    \n",
    "    # Make gender column two columns: \"M\" and \"F\" for men and women, both binary valued\n",
    "    df['M'] = df.loc[:,'Gender']\n",
    "    df['F'] = df.loc[:,'Gender'].apply(lambda x: 0 if x==1 else 1)\n",
    "    \n",
    "    # Drop columns we won't use anymore\n",
    "    df = df.drop(columns=['GPA','Gender'])\n",
    "    \n",
    "    # reorder columns so that sensitive features M and F are first\n",
    "    cols = df.columns.tolist()\n",
    "    cols_reorder = cols[-2:] + cols[0:-2]\n",
    "    outdf = df[cols_reorder]\n",
    "    \n",
    "    # Save final dataframe\n",
    "    outdf.to_csv(output_path_data,index=False,header=False)\n",
    "    print(f\"Saved data file to: {output_path_data}\\n\")\n",
    "\n",
    "    \n",
    "    # Save metadata json file\n",
    "    metadata_dict = {\n",
    "        \"regime\":\"supervised\",\n",
    "        \"sub_regime\":\"classification\",\n",
    "        \"all_col_names\":list(outdf.columns),\n",
    "        \"label_col_names\":[\"GPA_class\"],\n",
    "        \"sensitive_col_names\":[\"M\",\"F\"]\n",
    "    }\n",
    "    \n",
    "    with open(output_path_metadata,'w') as outfile:\n",
    "        json.dump(metadata_dict,outfile,indent=2)\n",
    "    print(f\"Saved metadata file to: {output_path_metadata}\\n\")\n",
    "    print(\"Here is what the final dataframe looks like:\")\n",
    "    return outdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0f771f0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved data file to: gpa_classification_dataset.csv\n",
      "\n",
      "Saved metadata file to: metadata_classification.json\n",
      "\n",
      "Here is what the final dataframe looks like:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>M</th>\n",
       "      <th>F</th>\n",
       "      <th>Physics</th>\n",
       "      <th>Biology</th>\n",
       "      <th>History</th>\n",
       "      <th>Second_Language</th>\n",
       "      <th>Geography</th>\n",
       "      <th>Literature</th>\n",
       "      <th>Portuguese_and_Essay</th>\n",
       "      <th>Math</th>\n",
       "      <th>Chemistry</th>\n",
       "      <th>GPA_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.403617</td>\n",
       "      <td>-0.760038</td>\n",
       "      <td>-1.495508</td>\n",
       "      <td>1.538450</td>\n",
       "      <td>0.982363</td>\n",
       "      <td>-0.282169</td>\n",
       "      <td>1.839716</td>\n",
       "      <td>1.326415</td>\n",
       "      <td>-0.551947</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.331064</td>\n",
       "      <td>-0.769699</td>\n",
       "      <td>-1.849386</td>\n",
       "      <td>-0.517353</td>\n",
       "      <td>-0.465143</td>\n",
       "      <td>-1.464796</td>\n",
       "      <td>-0.269207</td>\n",
       "      <td>-1.744443</td>\n",
       "      <td>-0.740591</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.050287</td>\n",
       "      <td>-1.268307</td>\n",
       "      <td>-0.105791</td>\n",
       "      <td>-1.800977</td>\n",
       "      <td>-1.332853</td>\n",
       "      <td>-1.694733</td>\n",
       "      <td>-0.865314</td>\n",
       "      <td>-0.898882</td>\n",
       "      <td>-1.467084</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.569989</td>\n",
       "      <td>1.093821</td>\n",
       "      <td>-0.525901</td>\n",
       "      <td>0.110934</td>\n",
       "      <td>-0.441563</td>\n",
       "      <td>-0.666367</td>\n",
       "      <td>0.474694</td>\n",
       "      <td>1.783769</td>\n",
       "      <td>0.147533</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.073098</td>\n",
       "      <td>0.800255</td>\n",
       "      <td>0.600797</td>\n",
       "      <td>0.403666</td>\n",
       "      <td>1.057400</td>\n",
       "      <td>-0.731171</td>\n",
       "      <td>0.246463</td>\n",
       "      <td>0.017919</td>\n",
       "      <td>-0.380420</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43298</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.491287</td>\n",
       "      <td>0.527786</td>\n",
       "      <td>0.849912</td>\n",
       "      <td>-0.351245</td>\n",
       "      <td>0.755381</td>\n",
       "      <td>-0.036623</td>\n",
       "      <td>0.387372</td>\n",
       "      <td>0.017919</td>\n",
       "      <td>0.019690</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43299</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2.086524</td>\n",
       "      <td>2.792616</td>\n",
       "      <td>1.608722</td>\n",
       "      <td>0.548363</td>\n",
       "      <td>2.602420</td>\n",
       "      <td>0.898735</td>\n",
       "      <td>1.769147</td>\n",
       "      <td>1.759789</td>\n",
       "      <td>2.318364</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43300</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.933335</td>\n",
       "      <td>2.453804</td>\n",
       "      <td>1.604370</td>\n",
       "      <td>0.856751</td>\n",
       "      <td>1.948139</td>\n",
       "      <td>0.703676</td>\n",
       "      <td>1.273787</td>\n",
       "      <td>1.691252</td>\n",
       "      <td>2.349477</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43301</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.420858</td>\n",
       "      <td>-1.230650</td>\n",
       "      <td>-0.370933</td>\n",
       "      <td>0.577717</td>\n",
       "      <td>-1.699551</td>\n",
       "      <td>1.006491</td>\n",
       "      <td>0.371422</td>\n",
       "      <td>-1.602135</td>\n",
       "      <td>-0.551947</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43302</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.551989</td>\n",
       "      <td>-1.510710</td>\n",
       "      <td>-0.673650</td>\n",
       "      <td>-0.479135</td>\n",
       "      <td>0.196190</td>\n",
       "      <td>-2.164834</td>\n",
       "      <td>-0.145625</td>\n",
       "      <td>-1.143822</td>\n",
       "      <td>-0.671499</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43303 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       M  F   Physics   Biology   History  Second_Language  Geography  \\\n",
       "0      0  1  0.403617 -0.760038 -1.495508         1.538450   0.982363   \n",
       "1      1  0 -0.331064 -0.769699 -1.849386        -0.517353  -0.465143   \n",
       "2      1  0 -1.050287 -1.268307 -0.105791        -1.800977  -1.332853   \n",
       "3      0  1  1.569989  1.093821 -0.525901         0.110934  -0.441563   \n",
       "4      1  0  0.073098  0.800255  0.600797         0.403666   1.057400   \n",
       "...   .. ..       ...       ...       ...              ...        ...   \n",
       "43298  1  0 -0.491287  0.527786  0.849912        -0.351245   0.755381   \n",
       "43299  1  0  2.086524  2.792616  1.608722         0.548363   2.602420   \n",
       "43300  0  1  1.933335  2.453804  1.604370         0.856751   1.948139   \n",
       "43301  0  1 -0.420858 -1.230650 -0.370933         0.577717  -1.699551   \n",
       "43302  0  1 -0.551989 -1.510710 -0.673650        -0.479135   0.196190   \n",
       "\n",
       "       Literature  Portuguese_and_Essay      Math  Chemistry  GPA_class  \n",
       "0       -0.282169              1.839716  1.326415  -0.551947        0.0  \n",
       "1       -1.464796             -0.269207 -1.744443  -0.740591        0.0  \n",
       "2       -1.694733             -0.865314 -0.898882  -1.467084        0.0  \n",
       "3       -0.666367              0.474694  1.783769   0.147533        0.0  \n",
       "4       -0.731171              0.246463  0.017919  -0.380420        0.0  \n",
       "...           ...                   ...       ...        ...        ...  \n",
       "43298   -0.036623              0.387372  0.017919   0.019690        0.0  \n",
       "43299    0.898735              1.769147  1.759789   2.318364        1.0  \n",
       "43300    0.703676              1.273787  1.691252   2.349477        1.0  \n",
       "43301    1.006491              0.371422 -1.602135  -0.551947        0.0  \n",
       "43302   -2.164834             -0.145625 -1.143822  -0.671499        1.0  \n",
       "\n",
       "[43303 rows x 12 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_seldonian_dataset(input_path=f,\n",
    "    output_path_data=\"gpa_classification_dataset.csv\",\n",
    "    output_path_metadata=\"metadata_classification.json\",\n",
    "    columns_raw=columns_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b2d16fe",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "seldo",
   "language": "python",
   "name": "seldo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}