examples/CI_minimalPipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Composite Indicators: Minimal Pipeline\n",
    "\n",
    "## 1 Basic settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from cif import cif\n",
    "import pandas as pd\n",
    "import re\n",
    "import datetime\n",
    "import warnings\n",
    "from IPython.display import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 Check availability of X-13ARIMA-SEATS model\n",
    "\n",
    "The model can be downloaded from https://www.census.gov/srd/www/x13as/ and its directory needs to be added to the system variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(os.environ['X13PATH'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 Settings\n",
    "\n",
    "Change the country of interest and other default settings here. For the complete list of available country codes run\n",
    "\n",
    "```python\n",
    "cif.getOECDJSONStructure(dsname = 'MEI', showValues = [0])\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "country = 'CZE' # Select target country\n",
    "\n",
    "#os.chdir('C:/path/') # Set path to to folder, where the plots and logs should be saved (optional)\n",
    "\n",
    "bw = False # True for black and white visualisations\n",
    "\n",
    "saveData = True # Save the original data sets if True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.4 Output directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "strDate = datetime.datetime.now().strftime(\"%Y-%m-%d-%H-%M\")\n",
    "\n",
    "outputDir = os.path.join('plots_' + country + '_' + strDate)\n",
    "os.makedirs(outputDir, exist_ok = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 Data Load\n",
    "\n",
    "Loading data from OECD API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all, subjects_all, measures_all = cif.createDataFrameFromOECD(countries = [country], dsname = 'MEI', frequency = 'M')\n",
    "data_rs, subjects_rs, measures_rs = cif.createDataFrameFromOECD(countries = [country], dsname = 'QNA', subject = ['B1_GE'], frequency = 'Q')\n",
    "\n",
    "print('Downloaded MEI data set size: %d x %d' % (data_all.shape[0], data_all.shape[1]))\n",
    "print('Downloaded reference data set size: %d x %d' % (data_rs.shape[0], data_rs.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the data\n",
    "\n",
    "if saveData:\n",
    "\n",
    "    data_all.to_csv(os.path.join(outputDir, 'data_all.csv'))\n",
    "    subjects_all.to_csv(os.path.join(outputDir, 'subjects_all.csv'))\n",
    "    measures_all.to_csv(os.path.join(outputDir, 'measures_all.csv'))\n",
    "    data_rs.to_csv(os.path.join(outputDir, 'data_rs.csv'))\n",
    "    subjects_rs.to_csv(os.path.join(outputDir, 'subjects_rs.csv'))\n",
    "    measures_rs.to_csv(os.path.join(outputDir, 'measures_rs.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all.tail(12) # MEI database data from last year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Leading indicators: Component series\n",
    "\n",
    "colMultiInd = data_all.columns.names.index('subject')\n",
    "\n",
    "ind_LOCO = subjects_all['id'].apply(lambda x: re.search(r'\\bLOCO', x) != None)\n",
    "subjects_LOCO = subjects_all[ind_LOCO]\n",
    "\n",
    "\n",
    "# Leading indicators: Reference series\n",
    "\n",
    "ind_LORS = subjects_all['id'].apply(lambda x: re.search(r'\\bLORS', x) != None)\n",
    "subjects_LORS = subjects_all[ind_LORS]\n",
    "\n",
    "\n",
    "# Leading indicators: CLI\n",
    "\n",
    "ind_LOLI = subjects_all['id'].apply(lambda x: re.search(r'\\bLOLI', x) != None)\n",
    "subjects_LOLI = subjects_all[ind_LOLI]\n",
    "\n",
    "\n",
    "# Candidate time series\n",
    "\n",
    "subjects_adj = subjects_all[-(ind_LOCO | ind_LORS | ind_LOLI)]\n",
    "data_adj = data_all.loc[ : , [x for x in data_all.columns if x[colMultiInd] in list(subjects_adj['id'])]].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 Data Transformations\n",
    "\n",
    "### 3.1 Reference Series\n",
    "\n",
    "#### 3.1.1 Priority list of reference series (GDP) and frequency conversion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rsPriorityList = [ 'LNBQRSA' # Best fit with OECD reference series\n",
    "                , 'CQR'\n",
    "                , 'LNBQR'\n",
    "                , 'DNBSA'\n",
    "                , 'DOBSA'\n",
    "                , 'CQRSA'\n",
    "                , 'CARSA'\n",
    "                , 'GPSA'\n",
    "                , 'GYSA'\n",
    "                , 'CPCARSA'\n",
    "                , 'VIXOBSA'\n",
    "                , 'VOBARSA'\n",
    "                , 'VPVOBARSA'\n",
    "                , 'HCPCARSA'\n",
    "                , 'HVPVOBARSA'\n",
    "                ]\n",
    "\n",
    "if (data_rs.shape[0] > 0):\n",
    "    \n",
    "    rsq = cif.getOnlyBestMeasure(df = data_rs, priorityList = rsPriorityList)\n",
    "    rsq = cif.getRidOfMultiindex(df = rsq)\n",
    "    rsq = cif.renameQuarterlyIndex(df = rsq)\n",
    "    rsq = cif.getIndexAsDate(df = rsq)\n",
    "    rs = cif.createMonthlySeries(df = rsq)\n",
    "    rs.dropna(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_rs.tail(4) # all available measures of the reference series (last year, quaterly series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.tail(12) # selected measure of the reference series (last year, monthly series)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1.2 Seasonal adjustment, outlier filtering and short-term prediction & Cycle identification (Hodrick-Prescott filter) & Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileLogs = open(os.path.join(outputDir, country + '_fileLogs_rsTransformation.txt'), 'w')\n",
    "rs_SA_HP_norm = cif.pipelineTransformations(rs, showPlots = False, savePlots = outputDir, saveLogs = fileLogs)\n",
    "fileLogs.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Individual indicators\n",
    "\n",
    "#### 3.2.1 Priority list of OECD available measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "priorityList = ['NCML'\n",
    "                , 'ML'\n",
    "                , 'CXML'\n",
    "                , 'ST'\n",
    "                , 'NCCU'\n",
    "                , 'CXCU'\n",
    "                , 'IXOB'\n",
    "                , 'NCMLSA'\n",
    "                , 'MLSA'\n",
    "                , 'CXMLSA'\n",
    "                , 'STSA'\n",
    "                , 'NCCUSA'\n",
    "                , 'CXCUSA'\n",
    "                , 'IXOBSA'\n",
    "                , 'IXNSA'\n",
    "                , 'GP'\n",
    "                , 'GY']\n",
    "\n",
    "if data_adj.shape[0] > 0:\n",
    "    \n",
    "    data = cif.getOnlyBestMeasure(df = data_adj, priorityList = priorityList)\n",
    "    data = cif.getRidOfMultiindex(df = data)\n",
    "    data = cif.getIndexAsDate(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.2.2 Seasonal adjustment, outlier filtering and short-term prediction & Cycle identification (Hodrick-Prescott filter) & Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with warnings.catch_warnings():\n",
    "            \n",
    "    warnings.simplefilter(\"ignore\")\n",
    "            \n",
    "    fileLogs = open(os.path.join(outputDir, 'fileLogs_dataTransformation.txt'), 'w')\n",
    "    data_SA_HP_norm = cif.pipelineTransformations(df = data, showPlots = False, savePlots = outputDir, saveLogs = fileLogs, createInverse = True) \n",
    "    fileLogs.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 Turning-point detection (Bry-Boschan algorithm)\n",
    "\n",
    "### 4.1 Reference series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileLogs = open(os.path.join(outputDir, country + '_fileLogs_rsEvaluation.txt'), 'w')\n",
    "rs_ind_turningPoints = cif.pipelineTPDetection(df = rs_SA_HP_norm, printDetails = False, showPlots = False, savePlots = outputDir, saveLogs = fileLogs)\n",
    "fileLogs.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Image(os.path.join(outputDir, country + '_B1_GE_LNBQRSA' + '_05_ext.png'), width = 600) # change name of the series here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 Individual indicators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileLogs = open(os.path.join(outputDir, 'fileLogs_dataEvaluation.txt'), 'w')\n",
    "data_ind_turningPoints = cif.pipelineTPDetection(df = data_SA_HP_norm, origColumns = list(data.columns), printDetails = False, showPlots = False, savePlots = outputDir, saveLogs = fileLogs)\n",
    "fileLogs.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Image(os.path.join(outputDir, country + '_XTEXVA01_NCML' + '_05_ext.png'), width = 600) # change name of the series here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5 Turning-points matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileLogs = open(os.path.join(outputDir, country + '_fileLogs_tpMatching.txt'), 'w')\n",
    "data_ind_extOrd, data_ind_time, data_ind_missing, data_ind_missingEarly, data_ind_extra = cif.pipelineTPMatching(df1 = rs_SA_HP_norm, df2 = data_SA_HP_norm, ind1 = rs_ind_turningPoints, ind2 = data_ind_turningPoints, printDetails = False, showPlots = False, savePlots = outputDir, saveLogs = fileLogs, nameSuffix = '_06_matching' + '_rs' + country)\n",
    "fileLogs.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Image(os.path.join(outputDir, country + '_XTEXVA01_NCML' + '_06_matching_rs' + country + '.png'), width = 600) # change name of the series here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6 Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_totalEval, data_selectedEval, data_selectedCol = cif.pipelineEvaluation(df1 = rs_SA_HP_norm, df2 = data_SA_HP_norm, missing = data_ind_missing, missingEarly = data_ind_missingEarly, extra = data_ind_extra, time = data_ind_time, maxInd = 15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_selectedEval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7 Aggregation & final evaluation \n",
    "\n",
    "### 7.1 CLI construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_cMat = data_SA_HP_norm.loc[:, data_selectedCol] # value of the de-trended, smoothed and normalised component\n",
    "\n",
    "CLI = cif.pipelineCreateCLI(agg_cMat).rename(columns = {'CLI': country + '_CLI'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cif.compareTwoSeries(CLI, rs_SA_HP_norm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.2 CLI turning points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileLogs = open(os.path.join(outputDir, country + '_fileLogs_CLIEvaluation.txt'), 'w')\n",
    "CLI_ind_turningPoints = cif.pipelineTPDetection(CLI, printDetails = False, showPlots = False, savePlots = outputDir, saveLogs = fileLogs)\n",
    "fileLogs.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.3 Match turning points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLI_ind_extOrd, CLI_ind_time, CLI_ind_missing, CLI_ind_missingEarly, CLI_ind_extra = cif.pipelineTPMatching(df1 = rs_SA_HP_norm, df2 = CLI, ind1 = rs_ind_turningPoints, ind2 = CLI_ind_turningPoints, showPlots = False, savePlots = outputDir, nameSuffix = '_06_matching' + '_rs' + country, bw = bw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Image(os.path.join(outputDir, country + '_CLI' + '_06_matching_rs' + country + '.png'), width = 600) # change name of the series here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.4 Basic characteristics of created CLI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLI_eval = cif.pipelineEvaluation(df1 = rs_SA_HP_norm, df2 = CLI, missing = CLI_ind_missing, missingEarly = CLI_ind_missingEarly, extra = CLI_ind_extra, time = CLI_ind_time, evalOnly = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLI_eval"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}