01-Data-Preparation-02-Data-Completion.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 01 Data Preparation\n",
    "\n",
    "To the pre-prepared COMPAS data we add predictions for name gender and origin made as made by NamSor API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# >>> Import Libraries\n",
    "\n",
    "print(\"Importing necessary libraries... \")\n",
    "\n",
    "import openapi_client #NamSor, see https://github.com/namsor/namsor-python-sdk2\n",
    "from openapi_client.rest import ApiException\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "print(\"Libraries imported.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# >>> Import COMPAS data set\n",
    "\n",
    "print(\"Importing COMPAS data set... \")\n",
    "\n",
    "df = pd.read_csv(\"data/compas_for_namsor.csv\")\n",
    "\n",
    "print(\"Data set imported. It is has {} entries and looks like this:\".format(df.shape[0]))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# >>> Preparing for API use\n",
    "\n",
    "# Get private API Key for NamSor API v2 (contained in txt file)\n",
    "print(\"Getting private key... \")\n",
    "\n",
    "key = ''\n",
    "\n",
    "try:\n",
    "    with open(\"key.txt\", \"r\") as file:\n",
    "        key = file.read()\n",
    "    if(len(key) < 0):\n",
    "        raise FileNotFoundError()\n",
    "except (FileNotFoundError):\n",
    "    print(\"Could not find private key. Please make sure you have an API key that you stored as key.txt in the root folder.\")\n",
    "\n",
    "print(\"Got private key.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Setting up NamSor API v2 connection settings...\")\n",
    "\n",
    "# Configure API key authorization: api_key\n",
    "configuration = openapi_client.Configuration()\n",
    "configuration.api_key['X-API-KEY'] = key\n",
    "\n",
    "# create an instance of the personal API class\n",
    "pers_api_instance = openapi_client.PersonalApi(openapi_client.ApiClient(configuration))\n",
    "\n",
    "print(\"Connection set.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# >>> Classifying names with NamSor API\n",
    "\n",
    "# Formatting a df of names\n",
    "print('Formatting names dataframe...')\n",
    "\n",
    "names_df = df[['entity_id', 'first', 'last']]\n",
    "\n",
    "print('Names dataframe formatted. It looks like this: ')\n",
    "print(names_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_first_last_name_geo_in(row) :\n",
    "    ''' This function turns a tuple of values [id, first_name, last_name] into a to_first_last_name_geo_in object'''\n",
    "    # https://github.com/namsor/namsor-python-sdk2/blob/master/docs/FirstLastNameGeoIn.md\n",
    "    if(not row[0] or not row[1] or not row[2]):\n",
    "        print(\"Entered invalid data to be turned into to_first_last_name_geo_in\")\n",
    "        return\n",
    "    return openapi_client.FirstLastNameGeoIn(id=row[0], \n",
    "                                         first_name=row[1], \n",
    "                                         last_name=row[2], \n",
    "                                         country_iso2='us') # http://www.vas.com/Tnotes/Country%20Codes.htm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Formatting a list of batches from the names df so names can be fed to the API batch-wise\n",
    "print('Creating list of name-batches...')\n",
    "\n",
    "names_stack = list() # this will be a list of name-batches generated from the df\n",
    "\n",
    "limit = len(names_df.index)\n",
    "start = 0\n",
    "end = -1\n",
    "batch_size = 1000 #1000 is the API limit given by NamSor\n",
    "\n",
    "while(end < limit):\n",
    "    start = end + 1\n",
    "    \n",
    "    try_end = start + batch_size\n",
    "    if(try_end <= limit):\n",
    "        end = try_end\n",
    "    else:\n",
    "        end = limit\n",
    "    \n",
    "    # each list item will fit openapi_client.BatchFirstLastNameGeoIn\n",
    "    current_df_batch = names_df[start:end]\n",
    "    \n",
    "    # https://stackoverflow.com/questions/16476924/how-to-iterate-over-rows-in-a-dataframe-in-pandas/55557758#55557758\n",
    "    list_first_last_name_geo_in = [to_first_last_name_geo_in(row) for row in current_df_batch[['entity_id', 'first', 'last']].to_numpy()]\n",
    "    names_stack.append(list_first_last_name_geo_in)\n",
    "    \n",
    "print('List of batches created.')\n",
    "\n",
    "print('Will need to make {} calls.'.format(len(names_stack)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(list_first_last_name_geo_in):\n",
    "    return openapi_client.BatchFirstLastNameGeoIn(personal_names=list_first_last_name_geo_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_gender_batch(list_first_last_name_geo_in):\n",
    "    api_response = pers_api_instance.gender_geo_batch(batch_first_last_name_geo_in=batch_first_last_name_geo_in)# call api\n",
    "    return api_response.personal_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_ethnicity_batch(list_first_last_name_geo_in):\n",
    "    # \"Output is W_NL (white, non latino), HL (hispano latino), A (asian, non latino), B_NL (black, non latino).\"\n",
    "    api_response = pers_api_instance.us_race_ethnicity_batch(batch_first_last_name_geo_in=batch_first_last_name_geo_in)# call api\n",
    "    return api_response.personal_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sending in one batch at a time and saving the result answer by answer.\n",
    "\n",
    "print(\"Sending batches to the API...\")\n",
    "\n",
    "result_gender = []\n",
    "result_ethnicity = []\n",
    "current = 0\n",
    "limit = len(names_stack)\n",
    "\n",
    "while(current < limit): # I assume len(result_gender) == len(result_ethnicity)\n",
    "    print(current)\n",
    "    batch_first_last_name_geo_in = get_batch(names_stack[current])\n",
    "    try:\n",
    "        result_gender.extend(predict_gender_batch(batch_first_last_name_geo_in))\n",
    "        result_ethnicity.extend(predict_ethnicity_batch(batch_first_last_name_geo_in))\n",
    "    except ApiException as e:\n",
    "        print(\"Exception when calling PersonalApi: {}\".format(e))\n",
    "        if(len(result_gender) != (batch_size * current + len(names_stack[current])) or\n",
    "          len(result_ethnicity) != (batch_size * current + len(names_stack[current]))):\n",
    "            print(\"Some names got lost when the exception at stack {} occurred. Please try again.\".format(current))\n",
    "            break\n",
    "        if(len(result_gender) == (batch_size * current + len(names_stack[current]))):\n",
    "            print(\"No names got lost for gender predictions. Trying again with stack size {}...\".format(len(names_stack)))\n",
    "        if(len(result_ethnicity) == (batch_size * current + len(names_stack[current]))):\n",
    "            print(\"No names got lost for ethnicity predictions. Trying again with stack size {}...\".format(len(names_stack)))\n",
    "        current -= 1\n",
    "        continue\n",
    "    current += 1\n",
    "\n",
    "print(\"All batches analyzed.\")\n",
    "print(result_gender[:5])\n",
    "print(result_ethnicity[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# >>> TODO: Save results to dataframe\n",
    "\n",
    "df.reset_index(inplace=True)\n",
    "df.set_index('entity_id', inplace=True)\n",
    "\n",
    "# Convert results (list of openapi_client.models.personal_name_gendered_out.PersonalNameGenderedOut) to (list of dictionaries)\n",
    "print('Filling the results into the names dataframe...')\n",
    "for i in range(len(result_gender)):\n",
    "    oapi_el = result_gender[i]\n",
    "    current_id = int(oapi_el.id)\n",
    "    df.loc[current_id, 'sex_pred'] = oapi_el.likely_gender\n",
    "    df.loc[current_id, 'sex_pred_prob'] = oapi_el.probability_calibrated\n",
    "    \n",
    "    oapi_el = result_ethnicity[i]\n",
    "    df.loc[current_id, 'race_pred'] = oapi_el.race_ethnicity\n",
    "    df.loc[current_id, 'race_pred_prob'] = oapi_el.probability_calibrated\n",
    "\n",
    "print('Dataframe completed with API results. Here are some results: {}'.format(df.head()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving results to 'names_cat.csv'\n",
    "print(\"Saving compas dataframe with predictions for gender and ethnicity to CSV... \")\n",
    "df.to_csv(\"data/compas_with_predictions.csv\")\n",
    "print(\"CSV saved!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}