Preparation_gpi_gdelt_files_for_machine_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the countries that have both GPI and Gdelt values \n",
    "\n",
    "countries_gpi = []\n",
    "for i in os.listdir('../../interpolated_gpi/'):\n",
    "    if i != '.DS_Store':\n",
    "        countries_gpi.append((i.split('_')[2].split('.')[0]))\n",
    "        \n",
    "countries_gdelt = []\n",
    "for j in os.listdir('../../gdelt_data_200803_202009/'):\n",
    "    if j != '.DS_Store':\n",
    "        countries_gdelt.append(j.split('.')[0])\n",
    "        \n",
    "all_countries = [country for country in countries_gdelt if country in countries_gpi] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(all_countries)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For every country create a dataframe with all the variables and the GPI values that we will use for machine learning. Repeat the process to create the dataframes for the future prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_variables_df = {}\n",
    "\n",
    "for country in all_countries:\n",
    "    print(country)\n",
    "    # Read the GPI interpolated data\n",
    "    gpi_data = pd.read_csv('../../interpolated_gpi/interpolated_gpi_%s.csv' %country)\n",
    "    gpi_data.astype({'MonthYear': 'str'}).dtypes #convert MonthYear to strings\n",
    "    all_variables_df[country] = pd.DataFrame({ 'MonthYear': gpi_data.MonthYear, 'GPI': gpi_data.GPI_score})\n",
    "    all_variables_df[country] = all_variables_df[country].set_index('MonthYear')\n",
    "    #first_gpiindex = all_variables_df[country].index[0]#Get the first date of the gpi that serves to leave out variables that will used only for prediction, since after a date we do not know the gpi\n",
    "    #last_gpiindex = all_variables_df[country].index[-1]#Get the last date of the gpi that serves to leave out variables that will used only for prediction, since after a date we do not know the gpi\n",
    "    \n",
    "    #Read the gdelt data as extracted from the Big Query\n",
    "    country_data = pd.read_csv('../../gdelt_data_200803_202009/%s.csv' %country)\n",
    "    # Remove event codes that are not numeric\n",
    "    country_data.EventBaseCode = pd.to_numeric(country_data.EventBaseCode, errors='coerce')\n",
    "    country_data = country_data.dropna()\n",
    "    country_data = country_data.reset_index(drop=True)\n",
    "    country_data.EventBaseCode =  country_data.EventBaseCode.astype(int) \n",
    "    variables = country_data['EventBaseCode'].unique()\n",
    "    #Create the dataframe to add only gdelt variables for prediction (only for the future prediction files)\n",
    "    all_variables_df_new[country] = pd.DataFrame()\n",
    "\n",
    "    for v in variables: #create small dataframes for each variable code and append in a bigger dataframe with all variables per country\n",
    "        \n",
    "        single_variable = country_data.loc[country_data.EventBaseCode == v]\n",
    "        single_variable = single_variable.reset_index(drop = True)\n",
    "        df_to_print = pd.DataFrame({'MonthYear': single_variable.MonthYear, 'event_count_' + str(v): single_variable.eventcount})\n",
    "        df_to_print = df_to_print.set_index('MonthYear')\n",
    "        #df_to_print = df_to_print.loc[first_gpiindex : last_gpiindex] #select the dataframe based on the index of gpi known values \n",
    "        all_variables_df[country] = pd.concat([all_variables_df[country], df_to_print], axis=1) \n",
    "        all_variables_df_new[country] = pd.concat([all_variables_df_new[country], df_new], axis=1) \n",
    "\n",
    "    #For machine learning\n",
    "    all_variables_df[country] = all_variables_df[country].fillna(0)\n",
    "    all_variables_df[country].to_csv('../../all_variables_and_GPI_monthly_all_countries/all_variables_%s.csv' %(country)) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}