prototype.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-07T01:27:10.983133Z",
     "start_time": "2017-05-07T01:27:10.944425"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from os import path, listdir\n",
    "\n",
    "from lxml import etree\n",
    "from lxml.etree import XPath\n",
    "import regex as re\n",
    "\n",
    "import ujson as json\n",
    "import pandas as pd\n",
    "\n",
    "from itertools import filterfalse\n",
    "from time import time\n",
    "from datetime import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:12:00.185703Z",
     "start_time": "2017-04-26T11:12:00.180410"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# hack to avoid passing the namespace all the time; from http://stackoverflow.com/a/17293795/786559\n",
    "# etree.FunctionNamespace(\"http://exslt.org/regular-expressions\").prefix = 're'\n",
    "re_NS = {'re': \"http://exslt.org/regular-expressions\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:12:02.198304Z",
     "start_time": "2017-04-26T11:12:02.183103"
    },
    "code_folding": [
     0
    ],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_next_file(journal_path, min_year=None, max_year=None):\n",
    "    \"\"\" Returns a the filename to process next, along with its year and month. \n",
    "        If the range is given, then only those years are considered \"\"\"\n",
    "    for year in listdir(journal_path):\n",
    "        # filter range:\n",
    "        if min_year and int(year) < min_year or \\\n",
    "           max_year and int(year) > max_year:\n",
    "            continue\n",
    "            \n",
    "        year_path = path.join(journal_path, year)\n",
    "        for filename in listdir(year_path):\n",
    "            month = path.splitext(filename)[0]\n",
    "            yield path.join(year_path, filename), int(year), int(month)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:12:03.535967Z",
     "start_time": "2017-04-26T11:12:03.529169"
    },
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_xpath_search(names, with_words=None):\n",
    "    \"\"\" Returns an XPath search function returning articles whose full_text matches any of their names\n",
    "    and any of `with_words` but none of `without_words` (if given)\"\"\"\n",
    "\n",
    "    names_regex_str = 're:test(., \"{regex}\", \"i\")'.format(regex='|'.join(names))\n",
    "    if with_words:\n",
    "        with_regex_str = ' and re:test(., \"{regex}\", \"i\")'.format(regex='|'.join(with_words))\n",
    "    else:\n",
    "        with_regex_str = ''\n",
    "    \n",
    "    match_string = './/full_text[{names_regex} {with_regex}]//ancestor::article'.format(\n",
    "            names_regex = names_regex_str, with_regex=with_regex_str)\n",
    "    return XPath(match_string, namespaces=re_NS)\n",
    "    # no need to pass the namespaces, because of the 'hack' above\n",
    "#     return etree.XPath(match_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:13:08.599688Z",
     "start_time": "2017-04-26T11:13:08.556259"
    },
    "code_folding": [],
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "result_path = '/home/tomoiaga/hum_dig/results/'\n",
    "\n",
    "# set the searches\n",
    "search_text = XPath('.//full_text/text()')\n",
    "search_date = XPath('./entity[1]/meta/issue_date/text()')\n",
    "search_journal = XPath('./entity[1]/meta/publication/text()')\n",
    "search_num_words = XPath('./entity[1]/meta/updated_word_count/text()')\n",
    "search_num_chars = XPath('./entity[1]/meta/updated_char_count/text()')\n",
    "\n",
    "def export_iramu(articles, country):\n",
    "    \"\"\" Appends all the articles to the country's file \"\"\"\n",
    "    # the current name is the first in the list\n",
    "    country_name = country['all_names'][0] \n",
    "    txt_name = path.join(result_path, country_name + '.txt')\n",
    "    \n",
    "    # pays, date, date_an, date_mois, date_jour, journal, len_mots, len_chars, colonist, continent\n",
    "    article_template = \"**** *p_{country} *date_{date} *an_{year} *mois_{month} *jour_{day} *j_{journal}\" + \\\n",
    "                           \" *lenMots_{words} *lenChar_{chars} *col_{colonist} *con_{continent}\\n\"\n",
    "    \n",
    "    # prepare country_name for iramuteq\n",
    "    country_name = country_name.replace('-', '_').replace(' ', '_').replace(\"'\", '_')\n",
    "        \n",
    "    with open(txt_name, 'a') as f:\n",
    "        for art in articles:\n",
    "            date_str = search_date(art)[0].replace('/', '-') # put in iramu format\n",
    "            date = dt.strptime(date_str, \"%d-%m-%Y\")\n",
    "            journal = search_journal(art)[0]\n",
    "            num_words = search_num_words(art)[0]\n",
    "            num_chars = search_num_chars(art)[0]\n",
    "            texts = search_text(art)\n",
    "            \n",
    "            # write the header line\n",
    "            try:\n",
    "                header = article_template.format(country=country_name, date=date_str, year=date.year, month=date.month, day=date.day,\n",
    "                                             journal=journal, words=num_words, chars=num_chars, \n",
    "                                             colonist=country['colonist'], continent=country['continent'])\n",
    "            except:\n",
    "                Tracer()()\n",
    "            f.write(header)\n",
    "            \n",
    "            # write the contents, removing any stars first\n",
    "            f.write(' '.join(texts).replace('*', ' '))\n",
    "            \n",
    "            # write separation\n",
    "            f.write('\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:13:15.682492Z",
     "start_time": "2017-04-26T11:13:15.664136"
    },
    "code_folding": [
     21
    ],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_countries(countries, span, with_words=[]):\n",
    "    \"\"\" Inserts the needed fields for searching \n",
    "        Returns the smallest and biggest independence years \"\"\"\n",
    "    min_year, max_year = 3000, 0\n",
    "    for name, country in countries.items():\n",
    "        # build the word list\n",
    "        country['all_names'] = [name]\n",
    "        if country['other']:\n",
    "            country['all_names'].append(country['other'])\n",
    "\n",
    "        # setup search\n",
    "        country['path'] = get_xpath_search(country['all_names'], with_words)\n",
    "        country['articles'] = []\n",
    "        \n",
    "        # update years\n",
    "        if country['ind_year'] < min_year:\n",
    "            min_year = country['ind_year']\n",
    "        if country['ind_year'] > max_year:\n",
    "            max_year = country['ind_year']\n",
    "    return min_year, max_year\n",
    "\n",
    "def year_in_range(country, year, span):\n",
    "    \"\"\" Whether we lookup the country in the given year, based on a span around its independence year \"\"\"\n",
    "    return country['ind_year'] - span <= year <= country['ind_year'] + span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:35:01.218045Z",
     "start_time": "2017-04-26T15:35:01.172106"
    },
    "code_folding": [],
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_articles(countries, with_words, span=2, year=None):\n",
    "    \"\"\" span = the period to consider around independence date\n",
    "        countries = { 'Lybie' : {           # each country is a dict\n",
    "                'other': 'Urundi',          # other names that it was known as \n",
    "                'ind_year' : 1960,          # year of independence\n",
    "                'continent': 'Africa',\n",
    "                'colonist' : 'France',\n",
    "                \n",
    "                # the function adds these:\n",
    "                'names'    : [...]\n",
    "                'path'     : XPath(names),  # an XLST search path for 'alt_names'\n",
    "                'articles' : []             # the list of articles corresponding to this country\n",
    "            }\n",
    "        } \"\"\"\n",
    "    # check you didn't pass the params in wrong order by mistake\n",
    "    assert(type(with_words) != 'number')\n",
    "    \n",
    "    if year:\n",
    "        min_year, max_year = year, year\n",
    "    else:\n",
    "        min_year, max_year = prepare_countries(countries, span, with_words)\n",
    "    \n",
    "    # augment with the required range\n",
    "    if span:\n",
    "        min_year, max_year = min_year - span, max_year + span\n",
    "    else:\n",
    "        min_year, max_year = 1945, 1995\n",
    "\n",
    "    root_path = \"/mnt/le_temps_data/letempsdata/data4-month/\"\n",
    "    log_file  = open('./logs/' + dt.now().strftime('%m_%d_%H_%M') + '.log', 'w')\n",
    "    for journal in ['GDL', 'JDG']:\n",
    "        journal_path = path.join(root_path, journal)\n",
    "        start = time()\n",
    "        prev_file_year = min_year - 1 # make sure it's different\n",
    "        year_articles = 0\n",
    "        \n",
    "        for file_path, file_year, file_month in get_next_file(journal_path, min_year, max_year):\n",
    "            # prin stats when finishing an file_year\n",
    "            if file_year != prev_file_year:\n",
    "                duration = time() - start\n",
    "                print(journal, prev_file_year, duration, 'Processed: ', year_articles, flush=True)\n",
    "                print(journal, prev_file_year, duration, 'Processed: ', year_articles, flush=True, file=log_file)\n",
    "                start = time()\n",
    "                prev_file_year = file_year\n",
    "                year_articles = 0\n",
    "\n",
    "            with open(file_path) as f:\n",
    "                tree = etree.parse(f)\n",
    "                \n",
    "                for name, country in countries.items():\n",
    "                    # skip irrelevant countries for this year\n",
    "                    if span and not year_in_range(country, file_year, span):\n",
    "                        continue\n",
    "\n",
    "                    articles = country['path'](tree)\n",
    "                    year_articles += len(articles)\n",
    "                    # save into the data structure\n",
    "                    # country['articles'].extend(articles)\n",
    "                    export_iramu(articles, country)\n",
    "    log_file.close()\n",
    "    # return countries\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "deco = {'décolonis' : {'colonist': 'tous', 'continent':'tous', 'ind_year': 1970, 'other':''}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1945 0.005806684494018555 Processed:  0\n",
      "GDL 1946 1.9948046207427979 Processed:  0\n",
      "GDL 1947 2.0686988830566406 Processed:  0\n",
      "GDL 1948 1.7058167457580566 Processed:  0\n",
      "GDL 1949 1.9859576225280762 Processed:  1\n",
      "GDL 1950 1.8600895404815674 Processed:  1\n",
      "GDL 1951 2.0267996788024902 Processed:  0\n",
      "GDL 1952 1.8184974193572998 Processed:  0\n",
      "GDL 1953 1.8471410274505615 Processed:  0\n",
      "GDL 1954 1.7810282707214355 Processed:  0\n",
      "GDL 1955 2.066211700439453 Processed:  0\n",
      "GDL 1956 1.817195177078247 Processed:  0\n",
      "GDL 1957 1.7490234375 Processed:  1\n",
      "GDL 1958 2.0494437217712402 Processed:  1\n",
      "GDL 1959 2.0210931301116943 Processed:  3\n",
      "GDL 1960 2.060258150100708 Processed:  5\n",
      "GDL 1961 2.326807737350464 Processed:  40\n",
      "GDL 1962 2.509708881378174 Processed:  81\n",
      "GDL 1963 2.3595192432403564 Processed:  59\n",
      "GDL 1964 2.3196139335632324 Processed:  45\n",
      "GDL 1965 2.6904420852661133 Processed:  35\n",
      "GDL 1966 2.0535528659820557 Processed:  19\n",
      "GDL 1967 2.6269397735595703 Processed:  41\n",
      "GDL 1968 2.2165913581848145 Processed:  21\n",
      "GDL 1969 1.8458168506622314 Processed:  7\n",
      "GDL 1970 2.0879249572753906 Processed:  12\n",
      "GDL 1971 2.603952407836914 Processed:  34\n",
      "GDL 1972 2.1445140838623047 Processed:  17\n",
      "GDL 1973 1.9711213111877441 Processed:  12\n",
      "GDL 1974 2.3708462715148926 Processed:  21\n",
      "GDL 1975 2.0496435165405273 Processed:  39\n",
      "GDL 1976 2.1810073852539062 Processed:  56\n",
      "GDL 1977 1.7189443111419678 Processed:  25\n",
      "GDL 1978 1.8145034313201904 Processed:  23\n",
      "GDL 1979 2.6524360179901123 Processed:  32\n",
      "GDL 1980 2.5091421604156494 Processed:  30\n",
      "GDL 1981 2.437422275543213 Processed:  22\n",
      "GDL 1982 2.585319995880127 Processed:  27\n",
      "GDL 1983 2.967111587524414 Processed:  30\n",
      "GDL 1984 2.9733223915100098 Processed:  28\n",
      "GDL 1985 3.2258639335632324 Processed:  26\n",
      "GDL 1986 3.264697313308716 Processed:  34\n",
      "GDL 1987 3.728832721710205 Processed:  52\n",
      "GDL 1988 3.984567880630493 Processed:  26\n",
      "GDL 1989 3.828756332397461 Processed:  29\n",
      "GDL 1990 2.961571455001831 Processed:  14\n",
      "GDL 1991 2.7920680046081543 Processed:  11\n",
      "GDL 1992 3.884608507156372 Processed:  20\n",
      "GDL 1993 4.8489344120025635 Processed:  16\n",
      "GDL 1994 3.626016616821289 Processed:  20\n",
      "GDL 1995 3.6168270111083984 Processed:  14\n",
      "JDG 1945 0.08615446090698242 Processed:  0\n",
      "JDG 1946 2.0353145599365234 Processed:  0\n",
      "JDG 1947 2.16247820854187 Processed:  0\n",
      "JDG 1948 2.0202624797821045 Processed:  0\n",
      "JDG 1949 2.112703561782837 Processed:  0\n",
      "JDG 1950 2.2179791927337646 Processed:  0\n",
      "JDG 1951 2.218139171600342 Processed:  0\n",
      "JDG 1952 2.3502297401428223 Processed:  0\n",
      "JDG 1953 2.5325424671173096 Processed:  0\n",
      "JDG 1954 2.4547505378723145 Processed:  0\n",
      "JDG 1955 2.731614589691162 Processed:  0\n",
      "JDG 1956 2.6711857318878174 Processed:  1\n",
      "JDG 1957 2.685168743133545 Processed:  0\n",
      "JDG 1958 2.802471876144409 Processed:  1\n",
      "JDG 1959 2.8850972652435303 Processed:  3\n",
      "JDG 1960 2.8231332302093506 Processed:  6\n",
      "JDG 1961 2.5686938762664795 Processed:  18\n",
      "JDG 1962 2.691423177719116 Processed:  91\n",
      "JDG 1963 2.8942408561706543 Processed:  56\n",
      "JDG 1964 2.861729621887207 Processed:  60\n",
      "JDG 1965 2.9897713661193848 Processed:  58\n",
      "JDG 1966 2.913914442062378 Processed:  49\n",
      "JDG 1967 3.140190839767456 Processed:  52\n",
      "JDG 1968 3.0111780166625977 Processed:  70\n",
      "JDG 1969 3.152552366256714 Processed:  49\n",
      "JDG 1970 3.1571826934814453 Processed:  40\n",
      "JDG 1971 3.007750988006592 Processed:  39\n",
      "JDG 1972 2.89385986328125 Processed:  24\n",
      "JDG 1973 2.7021005153656006 Processed:  28\n",
      "JDG 1974 3.0337092876434326 Processed:  37\n",
      "JDG 1975 2.9884908199310303 Processed:  58\n",
      "JDG 1976 3.0869288444519043 Processed:  88\n",
      "JDG 1977 2.9862442016601562 Processed:  45\n",
      "JDG 1978 3.026268243789673 Processed:  40\n",
      "JDG 1979 3.3106207847595215 Processed:  35\n",
      "JDG 1980 3.1453351974487305 Processed:  32\n",
      "JDG 1981 3.0205347537994385 Processed:  24\n",
      "JDG 1982 3.0968704223632812 Processed:  33\n",
      "JDG 1983 3.3615856170654297 Processed:  31\n",
      "JDG 1984 3.300243616104126 Processed:  25\n",
      "JDG 1985 3.393059492111206 Processed:  25\n",
      "JDG 1986 3.5033862590789795 Processed:  33\n",
      "JDG 1987 4.110381364822388 Processed:  44\n",
      "JDG 1988 4.142281770706177 Processed:  32\n",
      "JDG 1989 4.337376594543457 Processed:  27\n",
      "JDG 1990 4.423633098602295 Processed:  31\n",
      "JDG 1991 4.614504098892212 Processed:  18\n",
      "JDG 1992 4.816100120544434 Processed:  26\n",
      "JDG 1993 4.9251320362091064 Processed:  16\n",
      "JDG 1994 4.390565395355225 Processed:  27\n",
      "JDG 1995 4.7793288230896 Processed:  19\n"
     ]
    }
   ],
   "source": [
    "get_articles(deco, None, span=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Algérie': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1962-07-05 00:00:00'),\n",
       "  'ind_year': 1962,\n",
       "  'other': '',\n",
       "  'president': 'Ahmed Ben Bella'},\n",
       " 'Angola': {'colonist': 'Portugal',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1975-11-11 00:00:00'),\n",
       "  'ind_year': 1975,\n",
       "  'other': '',\n",
       "  'president': 'Agostinho Neto'},\n",
       " 'Bangladesh': {'colonist': 'Pakistan',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1971-03-26 00:00:00'),\n",
       "  'ind_year': 1971,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Birmanie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1948-01-04 00:00:00'),\n",
       "  'ind_year': 1948,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Botswana': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1966-09-30 00:00:00'),\n",
       "  'ind_year': 1966,\n",
       "  'other': 'Bechuanaland',\n",
       "  'president': 'Seretse Khama'},\n",
       " 'Burkina Faso': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-05 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Maurice Yaméogo'},\n",
       " 'Burundi': {'colonist': 'Belgique',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1962-07-01 00:00:00'),\n",
       "  'ind_year': 1962,\n",
       "  'other': 'Ruanda-Urundi',\n",
       "  'president': 'Mwambutsa IV'},\n",
       " 'Bénin': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Hubert Maga'},\n",
       " 'Cambodge': {'colonist': 'France',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1953-11-09 00:00:00'),\n",
       "  'ind_year': 1953,\n",
       "  'other': 'Indochine',\n",
       "  'president': ''},\n",
       " 'Cameroun': {'colonist': 'France Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-01-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Ahmadou Ahidjo'},\n",
       " 'Cap-Vert': {'colonist': 'Portugal',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1975-07-05 00:00:00'),\n",
       "  'ind_year': 1975,\n",
       "  'other': '',\n",
       "  'president': 'Aristides Pereira'},\n",
       " \"Côte d'Ivoire\": {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-07 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Félix Houphouët-Boigny'},\n",
       " 'Djibouti': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1977-06-27 00:00:00'),\n",
       "  'ind_year': 1977,\n",
       "  'other': 'Territoire français des Afars et des Issas',\n",
       "  'president': 'Hassan Gouled Aptidon'},\n",
       " 'Gabon': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-17 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Équatoriale française',\n",
       "  'president': 'Léon Mba'},\n",
       " 'Gambie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1965-02-18 00:00:00'),\n",
       "  'ind_year': 1965,\n",
       "  'other': '',\n",
       "  'president': 'Dawda Kairaba Jawara'},\n",
       " 'Ghana': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1957-03-06 00:00:00'),\n",
       "  'ind_year': 1957,\n",
       "  'other': \"Côte-de-l'Or\",\n",
       "  'president': 'Kwame Nkrumah'},\n",
       " 'Guinée': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1958-10-02 00:00:00'),\n",
       "  'ind_year': 1958,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Sékou Touré'},\n",
       " 'Guinée équatoriale': {'colonist': 'Espagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1968-10-12 00:00:00'),\n",
       "  'ind_year': 1968,\n",
       "  'other': 'Guinée espagnole',\n",
       "  'president': 'Francisco Macías Nguema'},\n",
       " 'Guinée-Bissau': {'colonist': 'Portugal',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1973-09-24 00:00:00'),\n",
       "  'ind_year': 1973,\n",
       "  'other': 'Guinée portugaise',\n",
       "  'president': 'Luís Cabral'},\n",
       " 'Inde': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1947-08-15 00:00:00'),\n",
       "  'ind_year': 1947,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Indonésie': {'colonist': 'Pays-Bas (Japon)',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1949-12-27 00:00:00'),\n",
       "  'ind_year': 1949,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Kenya': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1963-12-12 00:00:00'),\n",
       "  'ind_year': 1963,\n",
       "  'other': \"Afrique de l'Est Britannique\",\n",
       "  'president': 'Jomo Kenyatta'},\n",
       " 'Laos': {'colonist': 'France',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1949-07-19 00:00:00'),\n",
       "  'ind_year': 1949,\n",
       "  'other': 'Indochine',\n",
       "  'president': ''},\n",
       " 'Lesotho': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1966-10-04 00:00:00'),\n",
       "  'ind_year': 1966,\n",
       "  'other': 'Basutoland',\n",
       "  'president': 'Leabua Jonathan'},\n",
       " 'Libye': {'colonist': 'Italie',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1951-12-24 00:00:00'),\n",
       "  'ind_year': 1951,\n",
       "  'other': '',\n",
       "  'president': 'Idris al-Mahdi'},\n",
       " 'Madagascar': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-06-26 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Protectorat malgache',\n",
       "  'president': 'Philibert Tsiranana'},\n",
       " 'Malaisie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1957-08-31 00:00:00'),\n",
       "  'ind_year': 1957,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Malawi': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1964-07-06 00:00:00'),\n",
       "  'ind_year': 1964,\n",
       "  'other': 'Nyasaland',\n",
       "  'president': 'Hastings Kamuzu Banda'},\n",
       " 'Mali': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-09-22 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Modibo Keïta'},\n",
       " 'Maroc': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1956-04-07 00:00:00'),\n",
       "  'ind_year': 1956,\n",
       "  'other': '',\n",
       "  'president': 'Mohammed V'},\n",
       " 'Mauritanie': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-11-28 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Moktar Ould Daddah'},\n",
       " 'Mozambique': {'colonist': 'Portugal',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1975-06-25 00:00:00'),\n",
       "  'ind_year': 1975,\n",
       "  'other': 'Afrique orientale portugaise',\n",
       "  'president': 'Samora Machel'},\n",
       " 'Namibie': {'colonist': 'Afrique du Sud',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1990-03-21 00:00:00'),\n",
       "  'ind_year': 1990,\n",
       "  'other': 'Afrique du sud-ouest',\n",
       "  'president': 'Sam Nujoma'},\n",
       " 'Niger': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-03 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Hamani Diori'},\n",
       " 'Nigéria': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-10-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Abubakar Tafawa Balewa'},\n",
       " 'Ouganda': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1962-10-09 00:00:00'),\n",
       "  'ind_year': 1962,\n",
       "  'other': \"Afrique de l'Est Britannique\",\n",
       "  'president': 'Milton Obote'},\n",
       " 'Pakistan': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1947-08-14 00:00:00'),\n",
       "  'ind_year': 1947,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Philippines': {'colonist': 'Etats-Unis',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1946-07-04 00:00:00'),\n",
       "  'ind_year': 1946,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Rwanda': {'colonist': 'Belgique',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1962-07-01 00:00:00'),\n",
       "  'ind_year': 1962,\n",
       "  'other': 'Ruanda-Urundi',\n",
       "  'president': 'Grégoire Kayibanda'},\n",
       " 'République centrafricaine': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-13 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Équatoriale française',\n",
       "  'president': 'David Dacko'},\n",
       " 'Sahara occidental': {'colonist': 'Espagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1976-02-27 00:00:00'),\n",
       "  'ind_year': 1976,\n",
       "  'other': 'Sahara espagnol',\n",
       "  'president': 'Hassan II du Maroc'},\n",
       " 'Sierra Leone': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1961-04-27 00:00:00'),\n",
       "  'ind_year': 1961,\n",
       "  'other': '',\n",
       "  'president': 'Milton Margai'},\n",
       " 'Somalie': {'colonist': 'Italie Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-07-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Aden Abdullah Osman Daar'},\n",
       " 'Soudan': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1956-01-01 00:00:00'),\n",
       "  'ind_year': 1956,\n",
       "  'other': '',\n",
       "  'president': 'Ismail al-Azhari'},\n",
       " 'Sri Lanka': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1972-05-22 00:00:00'),\n",
       "  'ind_year': 1972,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Swaziland': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1968-09-06 00:00:00'),\n",
       "  'ind_year': 1968,\n",
       "  'other': '',\n",
       "  'president': 'Sobhuza II'},\n",
       " 'Sénégal': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-04-04 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Léopold Sédar Senghor'},\n",
       " 'Tanzanie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1961-12-09 00:00:00'),\n",
       "  'ind_year': 1961,\n",
       "  'other': 'Tanganyika',\n",
       "  'president': 'Julius Nyerere'},\n",
       " 'Tchad': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-11 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Équatoriale française',\n",
       "  'president': 'François Tombalbaye'},\n",
       " 'Togo': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-04-27 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Sylvanus Olympio'},\n",
       " 'Tunisie': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1956-03-20 00:00:00'),\n",
       "  'ind_year': 1956,\n",
       "  'other': '',\n",
       "  'president': 'Muhammad al-Amin Bey'},\n",
       " 'Viêt Nam': {'colonist': 'France',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1954-07-20 00:00:00'),\n",
       "  'ind_year': 1954,\n",
       "  'other': 'Indochine',\n",
       "  'president': ''},\n",
       " 'Zambie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1964-10-24 00:00:00'),\n",
       "  'ind_year': 1964,\n",
       "  'other': 'Rhodésie du nord',\n",
       "  'president': 'Kenneth Kaunda'},\n",
       " 'Zimbabwe': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1980-04-18 00:00:00'),\n",
       "  'ind_year': 1980,\n",
       "  'other': 'Rhodésie du Sud',\n",
       "  'president': 'Robert Mugabe'}}"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1944 0.005637407302856445 Processed:  0\n",
      "GDL 1945 1.9117045402526855 Processed:  57\n",
      "GDL 1946 3.7421064376831055 Processed:  1015\n",
      "GDL 1947 5.296717643737793 Processed:  1300\n",
      "GDL 1948 7.743750810623169 Processed:  1254\n",
      "GDL 1949 8.55806040763855 Processed:  1382\n",
      "GDL 1950 8.709542989730835 Processed:  1555\n",
      "GDL 1951 6.514414072036743 Processed:  455\n",
      "GDL 1952 6.849787712097168 Processed:  368\n",
      "GDL 1953 5.731336355209351 Processed:  473\n",
      "GDL 1954 6.122942924499512 Processed:  777\n",
      "GDL 1955 8.757981538772583 Processed:  2344\n",
      "GDL 1956 11.114814281463623 Processed:  1114\n",
      "GDL 1957 9.978607892990112 Processed:  806\n",
      "GDL 1958 9.422631740570068 Processed:  886\n",
      "GDL 1959 34.894047498703 Processed:  1640\n",
      "GDL 1960 33.89405703544617 Processed:  1188\n",
      "GDL 1961 44.93664526939392 Processed:  3164\n",
      "GDL 1962 45.990092039108276 Processed:  2572\n",
      "GDL 1963 47.97602987289429 Processed:  2366\n",
      "GDL 1964 18.895557403564453 Processed:  907\n",
      "GDL 1965 22.76979160308838 Processed:  714\n",
      "GDL 1966 10.86103892326355 Processed:  132\n",
      "GDL 1967 14.421532154083252 Processed:  96\n",
      "GDL 1968 7.873274803161621 Processed:  17\n",
      "GDL 1969 5.44373631477356 Processed:  16\n",
      "GDL 1970 4.189920663833618 Processed:  10\n",
      "GDL 1971 6.949308156967163 Processed:  12\n",
      "GDL 1972 4.1794257164001465 Processed:  10\n",
      "GDL 1973 3.825406312942505 Processed:  8\n",
      "GDL 1974 9.08236050605774 Processed:  99\n",
      "GDL 1975 8.492418050765991 Processed:  243\n",
      "GDL 1976 9.698123216629028 Processed:  304\n",
      "GDL 1977 7.009369134902954 Processed:  341\n",
      "GDL 1978 6.720126390457153 Processed:  228\n",
      "GDL 1979 8.513848066329956 Processed:  114\n",
      "GDL 1980 6.36521053314209 Processed:  147\n",
      "GDL 1981 3.8755040168762207 Processed:  108\n",
      "GDL 1982 3.9976987838745117 Processed:  73\n",
      "GDL 1983 4.514325141906738 Processed:  84\n",
      "GDL 1984 1.594287633895874 Processed:  0\n",
      "GDL 1985 1.596311092376709 Processed:  0\n",
      "GDL 1986 1.5792794227600098 Processed:  0\n",
      "GDL 1987 1.7486069202423096 Processed:  0\n",
      "GDL 1988 1.956239938735962 Processed:  0\n",
      "GDL 1989 5.865839719772339 Processed:  124\n",
      "GDL 1990 4.244382381439209 Processed:  119\n",
      "GDL 1991 4.099698305130005 Processed:  45\n",
      "GDL 1992 5.877241611480713 Processed:  43\n",
      "JDG 1944 0.014202594757080078 Processed:  0\n",
      "JDG 1945 2.0481648445129395 Processed:  94\n",
      "JDG 1946 3.9277195930480957 Processed:  1186\n",
      "JDG 1947 4.8759765625 Processed:  1069\n",
      "JDG 1948 7.93841290473938 Processed:  1406\n",
      "JDG 1949 8.181028127670288 Processed:  1423\n",
      "JDG 1950 8.400302171707153 Processed:  1476\n",
      "JDG 1951 6.66050124168396 Processed:  448\n",
      "JDG 1952 8.432703018188477 Processed:  497\n",
      "JDG 1953 7.546215057373047 Processed:  576\n",
      "JDG 1954 7.596323013305664 Processed:  905\n",
      "JDG 1955 10.400023460388184 Processed:  3358\n",
      "JDG 1956 13.894330263137817 Processed:  2372\n",
      "JDG 1957 13.898712635040283 Processed:  1444\n",
      "JDG 1958 11.577894687652588 Processed:  1273\n",
      "JDG 1959 43.154247760772705 Processed:  2446\n",
      "JDG 1960 42.2830970287323 Processed:  1268\n",
      "JDG 1961 45.10075569152832 Processed:  2987\n",
      "JDG 1962 47.634095191955566 Processed:  2758\n",
      "JDG 1963 54.59199261665344 Processed:  2709\n",
      "JDG 1964 21.80803108215332 Processed:  1014\n",
      "JDG 1965 23.416661024093628 Processed:  835\n",
      "JDG 1966 15.75505805015564 Processed:  231\n",
      "JDG 1967 16.14675283432007 Processed:  219\n",
      "JDG 1968 10.360045671463013 Processed:  37\n",
      "JDG 1969 9.277400732040405 Processed:  19\n",
      "JDG 1970 6.354232549667358 Processed:  32\n",
      "JDG 1971 7.2842652797698975 Processed:  11\n",
      "JDG 1972 5.660930156707764 Processed:  18\n",
      "JDG 1973 5.414292335510254 Processed:  19\n",
      "JDG 1974 11.53634786605835 Processed:  160\n",
      "JDG 1975 11.865099668502808 Processed:  367\n",
      "JDG 1976 13.105942249298096 Processed:  444\n",
      "JDG 1977 12.428276777267456 Processed:  559\n",
      "JDG 1978 11.56558346748352 Processed:  379\n",
      "JDG 1979 10.049048662185669 Processed:  136\n",
      "JDG 1980 7.721173048019409 Processed:  164\n",
      "JDG 1981 4.4522480964660645 Processed:  101\n",
      "JDG 1982 4.665002107620239 Processed:  79\n",
      "JDG 1983 4.957119703292847 Processed:  91\n",
      "JDG 1984 1.5409343242645264 Processed:  0\n",
      "JDG 1985 1.7730145454406738 Processed:  0\n",
      "JDG 1986 1.8045246601104736 Processed:  0\n",
      "JDG 1987 1.9879069328308105 Processed:  0\n",
      "JDG 1988 2.0269906520843506 Processed:  0\n",
      "JDG 1989 6.649320602416992 Processed:  136\n",
      "JDG 1990 6.68875527381897 Processed:  182\n",
      "JDG 1991 6.751617670059204 Processed:  72\n",
      "JDG 1992 7.442439556121826 Processed:  59\n"
     ]
    }
   ],
   "source": [
    "get_articles(countries, [], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Algérie': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1962-07-05 00:00:00'),\n",
       "  'ind_year': 1962,\n",
       "  'other': '',\n",
       "  'president': 'Ahmed Ben Bella'},\n",
       " 'Angola': {'colonist': 'Portugal',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1975-11-11 00:00:00'),\n",
       "  'ind_year': 1975,\n",
       "  'other': '',\n",
       "  'president': 'Agostinho Neto'},\n",
       " 'Bangladesh': {'colonist': 'Pakistan',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1971-03-26 00:00:00'),\n",
       "  'ind_year': 1971,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Birmanie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1948-01-04 00:00:00'),\n",
       "  'ind_year': 1948,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Botswana': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1966-09-30 00:00:00'),\n",
       "  'ind_year': 1966,\n",
       "  'other': 'Bechuanaland',\n",
       "  'president': 'Seretse Khama'},\n",
       " 'Burkina Faso': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-05 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Maurice Yaméogo'},\n",
       " 'Burundi': {'colonist': 'Belgique',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1962-07-01 00:00:00'),\n",
       "  'ind_year': 1962,\n",
       "  'other': 'Ruanda-Urundi',\n",
       "  'president': 'Mwambutsa IV'},\n",
       " 'Bénin': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Hubert Maga'},\n",
       " 'Cambodge': {'colonist': 'France',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1953-11-09 00:00:00'),\n",
       "  'ind_year': 1953,\n",
       "  'other': 'Indochine',\n",
       "  'president': ''},\n",
       " 'Cameroun': {'colonist': 'France Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-01-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Ahmadou Ahidjo'},\n",
       " 'Cap-Vert': {'colonist': 'Portugal',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1975-07-05 00:00:00'),\n",
       "  'ind_year': 1975,\n",
       "  'other': '',\n",
       "  'president': 'Aristides Pereira'},\n",
       " \"Côte d'Ivoire\": {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-07 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Félix Houphouët-Boigny'},\n",
       " 'Djibouti': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1977-06-27 00:00:00'),\n",
       "  'ind_year': 1977,\n",
       "  'other': 'Territoire français des Afars et des Issas',\n",
       "  'president': 'Hassan Gouled Aptidon'},\n",
       " 'Gabon': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-17 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Équatoriale française',\n",
       "  'president': 'Léon Mba'},\n",
       " 'Gambie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1965-02-18 00:00:00'),\n",
       "  'ind_year': 1965,\n",
       "  'other': '',\n",
       "  'president': 'Dawda Kairaba Jawara'},\n",
       " 'Ghana': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1957-03-06 00:00:00'),\n",
       "  'ind_year': 1957,\n",
       "  'other': \"Côte-de-l'Or\",\n",
       "  'president': 'Kwame Nkrumah'},\n",
       " 'Guinée': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1958-10-02 00:00:00'),\n",
       "  'ind_year': 1958,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Sékou Touré'},\n",
       " 'Guinée équatoriale': {'colonist': 'Espagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1968-10-12 00:00:00'),\n",
       "  'ind_year': 1968,\n",
       "  'other': 'Guinée espagnole',\n",
       "  'president': 'Francisco Macías Nguema'},\n",
       " 'Guinée-Bissau': {'colonist': 'Portugal',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1973-09-24 00:00:00'),\n",
       "  'ind_year': 1973,\n",
       "  'other': 'Guinée portugaise',\n",
       "  'president': 'Luís Cabral'},\n",
       " 'Inde': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1947-08-15 00:00:00'),\n",
       "  'ind_year': 1947,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Indonésie': {'colonist': 'Pays-Bas (Japon)',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1949-12-27 00:00:00'),\n",
       "  'ind_year': 1949,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Kenya': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1963-12-12 00:00:00'),\n",
       "  'ind_year': 1963,\n",
       "  'other': \"Afrique de l'Est Britannique\",\n",
       "  'president': 'Jomo Kenyatta'},\n",
       " 'Laos': {'colonist': 'France',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1949-07-19 00:00:00'),\n",
       "  'ind_year': 1949,\n",
       "  'other': 'Indochine',\n",
       "  'president': ''},\n",
       " 'Lesotho': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1966-10-04 00:00:00'),\n",
       "  'ind_year': 1966,\n",
       "  'other': 'Basutoland',\n",
       "  'president': 'Leabua Jonathan'},\n",
       " 'Libye': {'colonist': 'Italie',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1951-12-24 00:00:00'),\n",
       "  'ind_year': 1951,\n",
       "  'other': '',\n",
       "  'president': 'Idris al-Mahdi'},\n",
       " 'Madagascar': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-06-26 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Protectorat malgache',\n",
       "  'president': 'Philibert Tsiranana'},\n",
       " 'Malaisie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1957-08-31 00:00:00'),\n",
       "  'ind_year': 1957,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Malawi': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1964-07-06 00:00:00'),\n",
       "  'ind_year': 1964,\n",
       "  'other': 'Nyasaland',\n",
       "  'president': 'Hastings Kamuzu Banda'},\n",
       " 'Mali': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-09-22 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Modibo Keïta'},\n",
       " 'Maroc': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1956-04-07 00:00:00'),\n",
       "  'ind_year': 1956,\n",
       "  'other': '',\n",
       "  'president': 'Mohammed V'},\n",
       " 'Mauritanie': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-11-28 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Moktar Ould Daddah'},\n",
       " 'Mozambique': {'colonist': 'Portugal',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1975-06-25 00:00:00'),\n",
       "  'ind_year': 1975,\n",
       "  'other': 'Afrique orientale portugaise',\n",
       "  'president': 'Samora Machel'},\n",
       " 'Namibie': {'colonist': 'Afrique du Sud',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1990-03-21 00:00:00'),\n",
       "  'ind_year': 1990,\n",
       "  'other': 'Afrique du sud-ouest',\n",
       "  'president': 'Sam Nujoma'},\n",
       " 'Niger': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-03 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Hamani Diori'},\n",
       " 'Nigéria': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-10-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Abubakar Tafawa Balewa'},\n",
       " 'Ouganda': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1962-10-09 00:00:00'),\n",
       "  'ind_year': 1962,\n",
       "  'other': \"Afrique de l'Est Britannique\",\n",
       "  'president': 'Milton Obote'},\n",
       " 'Pakistan': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1947-08-14 00:00:00'),\n",
       "  'ind_year': 1947,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Philippines': {'colonist': 'Etats-Unis',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1946-07-04 00:00:00'),\n",
       "  'ind_year': 1946,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Rwanda': {'colonist': 'Belgique',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1962-07-01 00:00:00'),\n",
       "  'ind_year': 1962,\n",
       "  'other': 'Ruanda-Urundi',\n",
       "  'president': 'Grégoire Kayibanda'},\n",
       " 'République centrafricaine': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-13 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Équatoriale française',\n",
       "  'president': 'David Dacko'},\n",
       " 'Sahara occidental': {'colonist': 'Espagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1976-02-27 00:00:00'),\n",
       "  'ind_year': 1976,\n",
       "  'other': 'Sahara espagnol',\n",
       "  'president': 'Hassan II du Maroc'},\n",
       " 'Sierra Leone': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1961-04-27 00:00:00'),\n",
       "  'ind_year': 1961,\n",
       "  'other': '',\n",
       "  'president': 'Milton Margai'},\n",
       " 'Somalie': {'colonist': 'Italie Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-07-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Aden Abdullah Osman Daar'},\n",
       " 'Soudan': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1956-01-01 00:00:00'),\n",
       "  'ind_year': 1956,\n",
       "  'other': '',\n",
       "  'president': 'Ismail al-Azhari'},\n",
       " 'Sri Lanka': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1972-05-22 00:00:00'),\n",
       "  'ind_year': 1972,\n",
       "  'other': '',\n",
       "  'president': ''},\n",
       " 'Swaziland': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1968-09-06 00:00:00'),\n",
       "  'ind_year': 1968,\n",
       "  'other': '',\n",
       "  'president': 'Sobhuza II'},\n",
       " 'Sénégal': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-04-04 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Occidentale française',\n",
       "  'president': 'Léopold Sédar Senghor'},\n",
       " 'Tanzanie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1961-12-09 00:00:00'),\n",
       "  'ind_year': 1961,\n",
       "  'other': 'Tanganyika',\n",
       "  'president': 'Julius Nyerere'},\n",
       " 'Tchad': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-11 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique-Équatoriale française',\n",
       "  'president': 'François Tombalbaye'},\n",
       " 'Togo': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-04-27 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Sylvanus Olympio'},\n",
       " 'Tunisie': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1956-03-20 00:00:00'),\n",
       "  'ind_year': 1956,\n",
       "  'other': '',\n",
       "  'president': 'Muhammad al-Amin Bey'},\n",
       " 'Viêt Nam': {'colonist': 'France',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1954-07-20 00:00:00'),\n",
       "  'ind_year': 1954,\n",
       "  'other': 'Indochine',\n",
       "  'president': ''},\n",
       " 'Zambie': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1964-10-24 00:00:00'),\n",
       "  'ind_year': 1964,\n",
       "  'other': 'Rhodésie du nord',\n",
       "  'president': 'Kenneth Kaunda'},\n",
       " 'Zimbabwe': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1980-04-18 00:00:00'),\n",
       "  'ind_year': 1980,\n",
       "  'other': 'Rhodésie du Sud',\n",
       "  'president': 'Robert Mugabe'}}"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1945 0.02464890480041504 Processed:  0\n",
      "GDL 1946 72.52974724769592 Processed:  196\n",
      "GDL 1947 86.55457782745361 Processed:  383\n",
      "GDL 1948 72.93958163261414 Processed:  417\n",
      "GDL 1949 81.36562395095825 Processed:  348\n",
      "GDL 1950 82.39210176467896 Processed:  490\n",
      "GDL 1951 83.25923609733582 Processed:  459\n",
      "GDL 1952 69.3944046497345 Processed:  217\n",
      "GDL 1953 71.64987540245056 Processed:  405\n",
      "GDL 1954 75.30775880813599 Processed:  603\n",
      "GDL 1955 82.45940351486206 Processed:  998\n",
      "GDL 1956 79.0343017578125 Processed:  661\n",
      "GDL 1957 71.34850692749023 Processed:  746\n",
      "GDL 1958 84.59223532676697 Processed:  993\n",
      "GDL 1959 81.58337593078613 Processed:  1033\n",
      "GDL 1960 81.81172299385071 Processed:  839\n",
      "GDL 1961 99.96001362800598 Processed:  1600\n",
      "GDL 1962 101.86950445175171 Processed:  1168\n",
      "GDL 1963 96.50038433074951 Processed:  967\n",
      "GDL 1964 94.40223956108093 Processed:  609\n",
      "GDL 1965 109.50357413291931 Processed:  755\n",
      "GDL 1966 80.90409421920776 Processed:  460\n",
      "GDL 1967 105.17990255355835 Processed:  585\n",
      "GDL 1968 87.83056139945984 Processed:  290\n",
      "GDL 1969 67.5283477306366 Processed:  229\n",
      "GDL 1970 78.2839298248291 Processed:  285\n",
      "GDL 1971 101.97824811935425 Processed:  492\n",
      "GDL 1972 76.63382768630981 Processed:  495\n",
      "GDL 1973 72.21560382843018 Processed:  329\n",
      "GDL 1974 89.56658577919006 Processed:  424\n",
      "GDL 1975 80.25144386291504 Processed:  517\n",
      "GDL 1976 83.84259247779846 Processed:  681\n",
      "GDL 1977 68.02333879470825 Processed:  524\n",
      "GDL 1978 61.40921187400818 Processed:  480\n",
      "GDL 1979 107.58474564552307 Processed:  703\n",
      "GDL 1980 98.06715655326843 Processed:  680\n",
      "GDL 1981 98.89949369430542 Processed:  555\n",
      "GDL 1982 102.3554277420044 Processed:  667\n",
      "GDL 1983 118.87202548980713 Processed:  553\n",
      "GDL 1984 117.92285370826721 Processed:  687\n",
      "GDL 1985 128.51429915428162 Processed:  758\n",
      "GDL 1986 132.53606915473938 Processed:  659\n",
      "GDL 1987 150.6322100162506 Processed:  700\n",
      "GDL 1988 162.95161652565002 Processed:  645\n",
      "GDL 1989 151.70844960212708 Processed:  755\n",
      "GDL 1990 100.38972568511963 Processed:  487\n",
      "GDL 1991 100.06256914138794 Processed:  435\n",
      "GDL 1992 156.39066815376282 Processed:  596\n",
      "GDL 1993 203.11994171142578 Processed:  785\n",
      "GDL 1994 136.66990542411804 Processed:  491\n",
      "GDL 1995 149.14208459854126 Processed:  466\n",
      "JDG 1945 0.14688372611999512 Processed:  0\n",
      "JDG 1946 69.65120482444763 Processed:  153\n",
      "JDG 1947 74.19116020202637 Processed:  234\n",
      "JDG 1948 73.16969299316406 Processed:  294\n",
      "JDG 1949 77.28390216827393 Processed:  285\n",
      "JDG 1950 78.57893776893616 Processed:  292\n",
      "JDG 1951 79.83274555206299 Processed:  340\n",
      "JDG 1952 84.56573939323425 Processed:  397\n",
      "JDG 1953 89.28801822662354 Processed:  326\n",
      "JDG 1954 92.47213172912598 Processed:  522\n",
      "JDG 1955 95.31474113464355 Processed:  818\n",
      "JDG 1956 93.8998122215271 Processed:  637\n",
      "JDG 1957 96.52449584007263 Processed:  797\n",
      "JDG 1958 99.25282645225525 Processed:  845\n",
      "JDG 1959 99.63329100608826 Processed:  921\n",
      "JDG 1960 101.25774908065796 Processed:  824\n",
      "JDG 1961 99.06967878341675 Processed:  1666\n",
      "JDG 1962 106.14376950263977 Processed:  1758\n",
      "JDG 1963 109.00913548469543 Processed:  1255\n",
      "JDG 1964 109.51652646064758 Processed:  1029\n",
      "JDG 1965 112.70145511627197 Processed:  930\n",
      "JDG 1966 117.16105699539185 Processed:  850\n",
      "JDG 1967 119.93415522575378 Processed:  912\n",
      "JDG 1968 118.71613550186157 Processed:  566\n",
      "JDG 1969 120.69787645339966 Processed:  554\n",
      "JDG 1970 124.0771152973175 Processed:  649\n",
      "JDG 1971 111.40479111671448 Processed:  627\n",
      "JDG 1972 112.19955658912659 Processed:  684\n",
      "JDG 1973 108.09153985977173 Processed:  593\n",
      "JDG 1974 119.75659966468811 Processed:  629\n",
      "JDG 1975 115.88365960121155 Processed:  684\n",
      "JDG 1976 113.91277146339417 Processed:  1004\n",
      "JDG 1977 121.15725994110107 Processed:  830\n",
      "JDG 1978 113.22307753562927 Processed:  769\n",
      "JDG 1979 130.2935812473297 Processed:  783\n",
      "JDG 1980 122.21733093261719 Processed:  786\n",
      "JDG 1981 121.28517818450928 Processed:  566\n",
      "JDG 1982 123.91925287246704 Processed:  747\n",
      "JDG 1983 128.62464118003845 Processed:  597\n",
      "JDG 1984 129.71375346183777 Processed:  637\n",
      "JDG 1985 133.00188541412354 Processed:  857\n",
      "JDG 1986 137.80308294296265 Processed:  761\n",
      "JDG 1987 163.9935278892517 Processed:  674\n",
      "JDG 1988 165.8097403049469 Processed:  683\n",
      "JDG 1989 168.88750886917114 Processed:  727\n",
      "JDG 1990 174.88237047195435 Processed:  868\n",
      "JDG 1991 177.94043231010437 Processed:  697\n",
      "JDG 1992 195.24162459373474 Processed:  789\n",
      "JDG 1993 202.35726690292358 Processed:  783\n",
      "JDG 1994 180.7619025707245 Processed:  675\n",
      "JDG 1995 193.21677231788635 Processed:  617\n"
     ]
    }
   ],
   "source": [
    "get_articles(countries, ['décolonis', 'indépend'], 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Redo Burkina Faso after updating the countries without the `-` in `Afrique-Occidentale Française`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:57:25.400279Z",
     "start_time": "2017-04-26T11:57:25.358338"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:57:50.393554Z",
     "start_time": "2017-04-26T11:57:50.389677"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs = {'Burkina Faso': countries['Burkina Faso']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:57:50.947973Z",
     "start_time": "2017-04-26T11:57:50.941733"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1960, 1960)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prepare_countries(cs, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T11:58:00.882052Z",
     "start_time": "2017-04-26T11:58:00.874721"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'all_names': ['Burkina Faso', 'Afrique Occidentale française'],\n",
       " 'articles': [],\n",
       " 'colonist': 'France',\n",
       " 'continent': 'Afrique',\n",
       " 'ind_date': Timestamp('1960-08-05 00:00:00'),\n",
       " 'ind_year': 1960,\n",
       " 'other': 'Afrique Occidentale française',\n",
       " 'path': .//full_text[re:test(., \"Burkina Faso|Afrique Occidentale française\", \"i\") ]//ancestor::article,\n",
       " 'president': 'Maurice Yaméogo'}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs['Burkina Faso']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T12:06:07.339462Z",
     "start_time": "2017-04-26T12:05:16.861365"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1957 0.42054295539855957 Processed:  0\n",
      "GDL 1958 4.364308595657349 Processed:  5\n",
      "GDL 1959 3.9516022205352783 Processed:  2\n",
      "GDL 1960 4.933775424957275 Processed:  5\n",
      "GDL 1961 5.128218650817871 Processed:  2\n",
      "JDG 1957 0.08987617492675781 Processed:  0\n",
      "JDG 1958 4.962111234664917 Processed:  6\n",
      "JDG 1959 7.23511815071106 Processed:  8\n",
      "JDG 1960 4.62612509727478 Processed:  7\n",
      "JDG 1961 4.879289865493774 Processed:  2\n"
     ]
    }
   ],
   "source": [
    "get_articles(cs, None, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Redo Bénin because it only got this name 15 years after independece. Modified keywords to include `Dahomey`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T12:15:03.742253Z",
     "start_time": "2017-04-26T12:15:03.714019"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T12:15:04.535444Z",
     "start_time": "2017-04-26T12:15:04.531163"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs = {'Bénin': countries['Bénin']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T12:15:07.100069Z",
     "start_time": "2017-04-26T12:15:07.091714"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Bénin': {'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique Occidentale française|Dahomey',\n",
       "  'president': 'Hubert Maga'}}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T12:15:09.102757Z",
     "start_time": "2017-04-26T12:15:09.096743"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1960, 1960)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prepare_countries(cs, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T12:15:12.577524Z",
     "start_time": "2017-04-26T12:15:12.571248"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Bénin': {'all_names': ['Bénin', 'Afrique Occidentale française|Dahomey'],\n",
       "  'articles': [],\n",
       "  'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-08-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique Occidentale française|Dahomey',\n",
       "  'path': .//full_text[re:test(., \"Bénin|Afrique Occidentale française|Dahomey\", \"i\") ]//ancestor::article,\n",
       "  'president': 'Hubert Maga'}}"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T12:16:22.938895Z",
     "start_time": "2017-04-26T12:15:25.839891"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1957 0.5299463272094727 Processed:  0\n",
      "GDL 1958 4.897727966308594 Processed:  26\n",
      "GDL 1959 4.789002418518066 Processed:  28\n",
      "GDL 1960 5.559689998626709 Processed:  66\n",
      "GDL 1961 5.751474857330322 Processed:  36\n",
      "JDG 1957 0.07489323616027832 Processed:  0\n",
      "JDG 1958 6.0157318115234375 Processed:  27\n",
      "JDG 1959 5.808963060379028 Processed:  28\n",
      "JDG 1960 5.841915607452393 Processed:  50\n",
      "JDG 1961 6.002334117889404 Processed:  48\n"
     ]
    }
   ],
   "source": [
    "get_articles(cs, None, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Redo India"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T13:44:37.636045Z",
     "start_time": "2017-04-26T13:44:37.599297"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T13:44:39.074067Z",
     "start_time": "2017-04-26T13:44:39.068822"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs = {'__Inde': countries['__Inde']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T13:44:44.043466Z",
     "start_time": "2017-04-26T13:44:44.036594"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1947, 1947)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prepare_countries(cs, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T13:44:51.966380Z",
     "start_time": "2017-04-26T13:44:51.961529"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs['__Inde']['all_names'] = ['Inde ']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T14:07:17.951836Z",
     "start_time": "2017-04-26T14:07:17.947053"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs['__Inde']['path'] = XPath('.//full_text[re:test(., \"Inde \") ]//ancestor::article', namespaces=re_NS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T14:07:50.855157Z",
     "start_time": "2017-04-26T14:07:50.849881"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'__Inde': {'all_names': ['Inde '],\n",
       "  'articles': [],\n",
       "  'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Asie',\n",
       "  'ind_date': Timestamp('1947-08-15 00:00:00'),\n",
       "  'ind_year': 1947,\n",
       "  'other': 'Inde',\n",
       "  'path': .//full_text[re:test(., \"Inde \") ]//ancestor::article,\n",
       "  'president': ''}}"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T14:08:23.658548Z",
     "start_time": "2017-04-26T14:08:09.755189"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1944 0.005445718765258789 Processed:  0\n",
      "GDL 1945 1.2764034271240234 Processed:  23\n",
      "GDL 1946 1.3514559268951416 Processed:  102\n",
      "GDL 1947 1.2024457454681396 Processed:  93\n",
      "GDL 1948 1.2639334201812744 Processed:  99\n",
      "JDG 1944 0.009688854217529297 Processed:  0\n",
      "JDG 1945 1.479936122894287 Processed:  41\n",
      "JDG 1946 1.4699525833129883 Processed:  105\n",
      "JDG 1947 1.4772369861602783 Processed:  100\n",
      "JDG 1948 1.4513263702392578 Processed:  138\n"
     ]
    }
   ],
   "source": [
    "get_articles(cs, None, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Redo Mali"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:28:50.355782Z",
     "start_time": "2017-04-26T15:28:50.309358"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:29:11.695868Z",
     "start_time": "2017-04-26T15:29:11.691888"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs = {'__Mali': countries['Mali']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T13:44:44.043466Z",
     "start_time": "2017-04-26T13:44:44.036594"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1947, 1947)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prepare_countries(cs, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-07T01:29:37.134513Z",
     "start_time": "2017-05-07T01:29:37.128620"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<regex.Match object; span=(9, 14), match='Mali.'>"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.search(r'Mali([\\s.,;]|en)', \"Anomalie Mali.\", flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:32:38.177335Z",
     "start_time": "2017-04-26T15:32:38.171907"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs['__Mali']['path'] = XPath('.//full_text[re:test(., \"Mali |Malien|Afrique Occidentale française\") ]//ancestor::article', namespaces=re_NS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:32:39.011200Z",
     "start_time": "2017-04-26T15:32:39.006303"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'__Mali': {'all_names': ['Mali |Malien'],\n",
       "  'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-09-22 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique Occidentale française',\n",
       "  'path': .//full_text[re:test(., \"Mali |Malien|Afrique Occidentale française\") ]//ancestor::article,\n",
       "  'president': 'Modibo Keïta'}}"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:36:01.211034Z",
     "start_time": "2017-04-26T15:35:36.982146"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1957 0.03450465202331543 Processed:  0\n",
      "GDL 1958 2.0556042194366455 Processed:  5\n",
      "GDL 1959 1.9187612533569336 Processed:  28\n",
      "GDL 1960 2.3292438983917236 Processed:  85\n",
      "GDL 1961 2.5516412258148193 Processed:  27\n",
      "JDG 1957 0.026546478271484375 Processed:  0\n",
      "JDG 1958 2.5712990760803223 Processed:  1\n",
      "JDG 1959 2.710376501083374 Processed:  39\n",
      "JDG 1960 2.3744850158691406 Processed:  84\n",
      "JDG 1961 2.719193696975708 Processed:  48\n"
     ]
    }
   ],
   "source": [
    "get_articles(cs, None, 2, year=cs['__Mali']['ind_year'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Redo Nigeria"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:41:06.442275Z",
     "start_time": "2017-04-26T15:41:06.414138"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:41:27.529804Z",
     "start_time": "2017-04-26T15:41:27.525354"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs = {'__Nigeria': countries['Nigéria']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:41:32.297044Z",
     "start_time": "2017-04-26T15:41:32.291006"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'__Nigeria': {'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-10-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'president': 'Abubakar Tafawa Balewa'}}"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:43:42.082327Z",
     "start_time": "2017-04-26T15:43:42.078066"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs['__Nigeria']['all_names'] = ['__Nigeria']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:44:49.384775Z",
     "start_time": "2017-04-26T15:44:49.380170"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cs['__Nigeria']['path'] = XPath('.//full_text[re:test(., \"Nig[é|e]ria\") ]//ancestor::article', namespaces=re_NS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:44:53.046051Z",
     "start_time": "2017-04-26T15:44:53.038412"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'__Nigeria': {'all_names': ['__Nigeria'],\n",
       "  'colonist': 'Grande-Bretagne',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': Timestamp('1960-10-01 00:00:00'),\n",
       "  'ind_year': 1960,\n",
       "  'other': '',\n",
       "  'path': .//full_text[re:test(., \"Nig[é|e]ria\") ]//ancestor::article,\n",
       "  'president': 'Abubakar Tafawa Balewa'}}"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T15:45:49.432637Z",
     "start_time": "2017-04-26T15:45:31.814836"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1957 0.005060911178588867 Processed:  0\n",
      "GDL 1958 1.3641092777252197 Processed:  19\n",
      "GDL 1959 1.4646339416503906 Processed:  31\n",
      "GDL 1960 1.6261332035064697 Processed:  94\n",
      "GDL 1961 1.7359566688537598 Processed:  63\n",
      "JDG 1957 0.0045850276947021484 Processed:  0\n",
      "JDG 1958 1.93929123878479 Processed:  34\n",
      "JDG 1959 1.990067481994629 Processed:  37\n",
      "JDG 1960 1.81256103515625 Processed:  88\n",
      "JDG 1961 1.9920704364776611 Processed:  103\n"
     ]
    }
   ],
   "source": [
    "get_articles(cs, None, 2, year=cs['__Nigeria']['ind_year'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Congo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Congo Kinshasa | Congo Leopoldville | Congo Belge"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Afrique Equatoriale française | Congo Brazza"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:31:55.800655Z",
     "start_time": "2017-04-26T16:31:55.759850"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:31:58.380064Z",
     "start_time": "2017-04-26T16:31:58.375026"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "countries = {name:d for name, d in countries.items() if name.startswith('Congo')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:32:05.618957Z",
     "start_time": "2017-04-26T16:32:05.609485"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1960, 1960)"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prepare_countries(countries, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:33:56.772492Z",
     "start_time": "2017-04-26T16:33:56.766360"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Congo belge': {'all_names': ['Congo belge',\n",
       "   'Congo-Kinshasa|Congo-Léopoldville'],\n",
       "  'articles': [],\n",
       "  'colonist': 'Belgique',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': '',\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Congo-Kinshasa|Congo-Léopoldville',\n",
       "  'path': .//full_text[re:test(., \"Congo belge|Congo-Kinshasa|Congo-Léopoldville\", \"i\") ]//ancestor::article,\n",
       "  'president': ''},\n",
       " 'Congo-Brazzaville': {'all_names': ['Congo-Brazzaville',\n",
       "   'Afrique [É|E]quatoriale française'],\n",
       "  'articles': [],\n",
       "  'colonist': 'France',\n",
       "  'continent': 'Afrique',\n",
       "  'ind_date': '',\n",
       "  'ind_year': 1960,\n",
       "  'other': 'Afrique [É|E]quatoriale française',\n",
       "  'path': .//full_text[re:test(., \"Congo-Brazzaville|Afrique [É|E]quatoriale française\", \"i\") ]//ancestor::article,\n",
       "  'president': ''}}"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:33:56.764170Z",
     "start_time": "2017-04-26T16:32:59.746995"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1957 0.005606412887573242 Processed:  0\n",
      "GDL 1958 4.7839930057525635 Processed:  28\n",
      "GDL 1959 4.809852123260498 Processed:  95\n",
      "GDL 1960 5.6952879428863525 Processed:  74\n",
      "GDL 1961 5.881765365600586 Processed:  24\n",
      "JDG 1957 0.004813194274902344 Processed:  0\n",
      "JDG 1958 5.923313617706299 Processed:  250\n",
      "JDG 1959 6.035631895065308 Processed:  138\n",
      "JDG 1960 5.843557119369507 Processed:  153\n",
      "JDG 1961 6.206706523895264 Processed:  96\n"
     ]
    }
   ],
   "source": [
    "get_articles(countries, None, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:40:19.516904Z",
     "start_time": "2017-04-26T16:40:19.510956"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "countries = {'Congo' : {'all_names': ['Congo'], 'articles':[], 'colonist':'_', 'ind_date':'', 'ind_year':1960,\n",
    "                       'other':'', 'president':'', 'continent':'Afrique', 'path':XPath('.//full_text[re:test(., \"Congo\", \"i\") ]//ancestor::article')}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:40:51.988399Z",
     "start_time": "2017-04-26T16:40:23.211322"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1957 0.005195140838623047 Processed:  0\n",
      "GDL 1958 2.367814779281616 Processed:  44\n",
      "GDL 1959 2.2984087467193604 Processed:  159\n",
      "GDL 1960 2.9160654544830322 Processed:  858\n",
      "GDL 1961 2.912484645843506 Processed:  802\n",
      "JDG 1957 0.003767251968383789 Processed:  0\n",
      "JDG 1958 3.0067803859710693 Processed:  327\n",
      "JDG 1959 3.172656536102295 Processed:  309\n",
      "JDG 1960 2.9996840953826904 Processed:  974\n",
      "JDG 1961 3.0875766277313232 Processed:  857\n"
     ]
    }
   ],
   "source": [
    "get_articles(countries, None, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### New Vietnam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:55:09.046942Z",
     "start_time": "2017-04-26T16:55:08.943246"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "countries = pd.read_excel('./Liste mots cles.xlsx', sheetname='Pays_2', index_col='name', convert_float=True).fillna('').to_dict('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:55:53.460093Z",
     "start_time": "2017-04-26T16:55:53.451624"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cs = {'Viêt Nam' : countries['Viêt Nam']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:56:16.379896Z",
     "start_time": "2017-04-26T16:56:16.374250"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cs['Viêt Nam']['ind_year'] = 1975"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:56:27.902147Z",
     "start_time": "2017-04-26T16:56:27.895070"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1975, 1975)"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prepare_countries(cs, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-26T16:57:29.445662Z",
     "start_time": "2017-04-26T16:56:42.693860"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDL 1972 0.02689957618713379 Processed:  0\n",
      "GDL 1973 4.287868976593018 Processed:  145\n",
      "GDL 1974 3.826289415359497 Processed:  36\n",
      "GDL 1975 4.056542158126831 Processed:  111\n",
      "GDL 1976 3.3563852310180664 Processed:  34\n",
      "JDG 1972 0.015259981155395508 Processed:  0\n",
      "JDG 1973 5.659358739852905 Processed:  199\n",
      "JDG 1974 5.4528985023498535 Processed:  44\n",
      "JDG 1975 5.608933687210083 Processed:  160\n",
      "JDG 1976 5.856219291687012 Processed:  51\n"
     ]
    }
   ],
   "source": [
    "get_articles(cs, None, 2)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}