2018/materials/boulder/day2-digital-trace-data/Day 2 - Wikipedia Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.options.display.max_columns = 100\n",
    "\n",
    "import requests\n",
    "import json\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import urlretrieve\n",
    "from urllib.parse import urlparse, quote, unquote\n",
    "from datetime import datetime\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "import networkx as nx \n",
    "import time, re\n",
    "\n",
    "_dir = 'E:/Dropbox/Workspace/Wikipedia_Trump/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://stackoverflow.com/a/312464/1574687\n",
    "\n",
    "def chunk_list(l,size=50): \n",
    "    \"\"\"Yield successive n-sized chunks from l.\"\"\"\n",
    "    chunk_list = list()\n",
    "    for i in range(0, len(l), size):\n",
    "        chunk_list.append(l[i:i + size])\n",
    "    return chunk_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Revisions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_page_revisions(page_title,lang='en',redirects=True):\n",
    "    \"\"\"Takes Wikipedia page title and returns a DataFrame of revisions\n",
    "    \n",
    "    page_title - a string with the title of the page on Wikipedia\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition,\n",
    "        defaults to \"en\"\n",
    "        \n",
    "    Returns:\n",
    "    revision_list - a list of dictionaries, where each dictionary is the revision\n",
    "        meta-data such as parentid, revid,sha1, size, timestamp, and user name\n",
    "    \"\"\"\n",
    "    \n",
    "    revision_list = list()\n",
    "    \n",
    "    if redirects:\n",
    "        redirects = 1\n",
    "    else:\n",
    "        redirects = 0\n",
    "    \n",
    "    # Configure the parameters\n",
    "    params = {'action':'query',\n",
    "              'prop':'revisions',\n",
    "              'titles':page_title,\n",
    "              'redirects':redirects,\n",
    "              'rvprop':'ids|userid|comment|timestamp|user|size|sha1',\n",
    "              'rvlimit':500,\n",
    "              'rvdir':'newer',\n",
    "              'format':'json',\n",
    "              'formatversion':2\n",
    "             }\n",
    "    \n",
    "    # Point to the API\n",
    "    url = 'https://{0}.wikipedia.org/w/api.php'.format(lang)\n",
    "    \n",
    "    # Make the request\n",
    "    json_response = requests.get(url,params=params).json()\n",
    "    \n",
    "    if 'query' in json_response:\n",
    "        subquery_revision_list = json_response['query']['pages'][0]['revisions']\n",
    "        revision_list += subquery_revision_list\n",
    "    else:\n",
    "        raise KeyError(\"There is no 'query' key present in the API response.\")\n",
    "    \n",
    "    while True:\n",
    "    \n",
    "        if 'continue' not in json_response:\n",
    "            break\n",
    "            \n",
    "        else:\n",
    "            query_continue = json_response['continue']['rvcontinue']\n",
    "            \n",
    "            continue_params = {'action':'query',\n",
    "                               'prop':'revisions',\n",
    "                               'titles':page_title,\n",
    "                               'redirects':redirects,\n",
    "                               'rvprop':'ids|userid|comment|timestamp|user|size|sha1',\n",
    "                               'rvcontinue':query_continue,\n",
    "                               'rvlimit':500,\n",
    "                               'rvdir':'newer',\n",
    "                               'format':'json',\n",
    "                               'formatversion':2\n",
    "                              }\n",
    "            \n",
    "             # Make the request\n",
    "            json_response = requests.get(url,params=params).json()\n",
    "\n",
    "            if 'query' in json_response:\n",
    "                subquery_revision_list = json_response['query']['pages'][0]['revisions']\n",
    "                revision_list += subquery_revision_list\n",
    "            else:\n",
    "                raise KeyError(\"There is no 'query' key present in the API response.\")\n",
    "    \n",
    "    df = pd.DataFrame(revision_list)\n",
    "    df['page'] = page_title\n",
    "    df['timestamp'] = pd.to_datetime(df['timestamp'])\n",
    "    df['date'] = df['timestamp'].apply(lambda x:x.date())\n",
    "    df['userid'] = df['userid'].fillna(0).apply(lambda x:str(int(x)))\n",
    "    df['diff'] = df['size'].diff()\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example from class\n",
    "\n",
    "Get page revisions for Wikipedia's NPOV policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "npov_revs = get_page_revisions('Wikipedia:Neutral point of view')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get page revisions for the talk page."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "npov_talk_revs = get_page_revisions('Wikipedia talk:Neutral point of view')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many changes have been made to the policy versus the policy discussion?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5315, 14735)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(npov_revs), len(npov_talk_revs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many unique users have contributed to the policy itself versus the policy discussion?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2050, 1266)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(npov_revs['user'].unique()), len(npov_talk_revs['user'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute some daily statistics so we can plot."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count the number of unique revisions in a day and the average page size on a day\n",
    "daily_df = npov_revs.groupby('date').agg({'sha1':pd.Series.nunique,'size':np.mean})\n",
    "\n",
    "# Reindex so that the dates are continuous\n",
    "daily_df = daily_df.reindex(pd.date_range(_s.index.min(),_s.index.max()))\n",
    "\n",
    "# For revisions, if we don't observe a revision on a day, then there were no revisions\n",
    "daily_df['sha1'].fillna(0,inplace=True)\n",
    "\n",
    "# For page size, if you don't observe a page size, then take the previous valid pagesize\n",
    "daily_df['size'].fillna(method='ffill',inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the page size over time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a27035a58>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD7CAYAAACIYvgKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYHFW5+PFv9+yTzEz2kH0h5GQCBAgJQUhIWEMIiAsiAtcFQdHoD66oIDcQEEQuAoKKV0QwoiIqIIgQZCcQQmIWQgLNCdmXyTaZzL5P9++P6u7pnumluru6q3r6/TwPD5Pq09VvV1edt86pU6dcPp8PIYQQuc1tdwBCCCHsJ8lACCGEJAMhhBCSDIQQQiDJQAghBJIMhBBCIMlACCEEkgyEEEIgyUAIIQSQb3cAZq1bt85XUlKS8nra2tooKiqyICJrOC0ecF5MTosHnBeT0+IB58XktHggMzE1NzdXn3zyyUPjlcuaZOByuaisrEx5PR6Px5L1WMVp8YDzYnJaPOC8mJwWDzgvJqfFA5mJae3atTvNlJNuIiGEEJIMhBBCSDIQQgiBJAMhhBBIMhBCCIEkAyGEEEgyEEIIgSSDnLXlYCMrtx62OwwhhENkzU1nwlrn3P8WADvuXmhzJEIIJ5CWgRBCCHMtA6XUj4BPA4XAr4G3gKWAD9gELNJae5VSS4CFQCdwvdZ6tVJqktmyVn4xIYQQ5sVtGSil5gGnAacDc4ExwP3AYq31HMAFXKyUmu5/fRZwGfCQfxWJlBVCCGEDM91E84GNwD+A54F/ASdjtA4AlgHnALOBl7XWPq31LiBfKTU0wbJCCCFsYKabaAgwDrgQmAD8E3BrrX3+1xuACqAcCB2eEljuSqDsoWhBeL1ePB6PiXBja21ttWQ9VrE7nkifbXdMPTktHnBeTE6LB5wXk9PiAWfFZCYZHAY+1lq3A1op1YrRVRRQBtQC9f6/ey73JlA2KrfbLVNYW2obQMTPlm0Un9Niclo84LyYnBYPZGwKa1PlzHQTvQOcr5RyKaVGAv2A1/zXEgAWAG8DK4D5Sim3UmosRuuhGlifQFkhhBA2iNsy0Fr/Syl1BrAaI3ksArYDjyilCgEP8JTWuksp9TawMqQcwA0JlBVCCGEDU0NLtdY/jLB4boRytwG39Vi22WxZIYQQ9pCbzoQQQkgyEEIIIclACCEEkgyEEEIgyUAIIQSSDIQQQiDJQAghBJIMhBBCIMlACCEEkgxEGmzYXcsnBxrsDkMIkQB5BrKw3MUPrQDk+cpCZBNpGQghhJBkIIQQQpKBEEIIJBkIIYRAkoEQQggkGQghhECSgRBCCCQZCCGEQJKBEEIIJBkIIYRAkoEQQggkGQghhECSgRBCCCQZCCGEQJKBEEIIJBkIIYRAkoEQQggkGQghhECSgRBCCCQZCCGEQJKBEEIIJBkIIYQA8s0UUkqtB+r8/9wOPAw8CHQCL2utb1dKuYFfAycAbcDVWustSqlTzZa18HsJIYRIQNxkoJQqBtBazwtZ9j7weWAb8IJSajowHijWWn/KnwDuAy4GfpNAWSGEEDYw0zI4AShVSr3sL38bUKS13gqglPo3cDYwAngJQGv9nlJqhlKq3GxZS7+VEEKIhJhJBs3AvcDvgGOAZUBtyOsNwESgnO6uJIAu/7J6M2WVUvla685oQXi9Xjwej4lwY2ttbbVkPVaxO55In21VTFZ9L7u3USROi8lp8YDzYnJaPOCsmMwkg83AFq21D9islKoDBoW8XoaRHEr9fwe4MRJBmZmysRIBgNvtprKy0kS4sXk8HkvWYxX74tkGEPGzU48p+rqT4bTfDJwXk9PiAefF5LR4IDMxrV271lQ5M6OJrsLo00cpNRKjIm9SSh2tlHIB84G3gRXABf5ypwIbtdb1QLuZsua/mhBCCKuZaRk8CixVSr0D+DCSgxf4M5CHMUJolVLqP8C5Sql3ARfwNf/7r02grBBCCBvETQZa63bg8ggvndqjnBej4u/5/vfMlhVCCGEPuelMCCGEJAMhhBCSDIQQQiDJQAghBJIMhBBCIMlACCEEkgyEEEIgyUAIIQSSDIQQQiDJQAghBJIMhBBCIMlACCEEkgxEDtp5uIm2zi67wxDCUSQZiJyyu6aZuT97E7X4JZ5au4eZP3mVQw1tdoclhO0kGYic8qvXtwT//v7fN3CooY2ZP3mVlnZpKYjcJslA5JTalvaIy/fXt2Y4EiGcRZKByCleX+TlPl+UF4TIEZIMRE6JVulLKhC5zswzkEUfsmZHDX/9z267w7BNV7SmgRA5TpJBDjlY38olv1lpdxi2it5NlNk4hHAa6SbKEbsON3PKXa/1Wv5RVT33vaxtiMge3qi1vmQDkdskGeSIFVurIy7/7K9X8MvXt9De6TW1noff2srUW18yXd5poiUDaRmIXCfJIEds3FsXcXmgD93lir+OZ9bt4afLPqa5vYs9R5qtDC8jqhvboiYxyQUi18k1gxzwzifVrNt5JGYZM2fG3/vbhu7yqQaVYT6fjxl3vhrj9QwGI4QDSTLo47748EpWba+JW84Xp3rvOQon2yrP9btrY74e7/ub4fX6cLtNNLGEcCDpJurDHl+5w1QigPiV+7tRrjkko665g417Indbpcsz6/bEfD3V5LavroWJN7/IX/+zK7UVOUBDawf3v6ypa+6wOxSRQZIM+rBbn/vQdNl4lWGemYsKJn3xtyu56FfvWLa+eFo7uvjTe7Er6VSTwc7DxjWUp9ftjVrmNc8BDmTBtBf/3FDFL17fwuMrd9gdisggSQZ9QENrB6u2HeZIkzHvjs/nY7XJFkFAvG6Sb/5xba93JOvj/Q1JvzdRv3t7G2ff91bccql2ExXkGcly9faaiK2QlvYuvv6HNcyKMLzXSfT+Bl73HATgL6t3yU16OUSSQR/wjcfX8sXfvse5P1+Oz+dj8bObuPRhczeXdfoP9it+typqmWUb99HQ1mlJrJnS5fWx6Il13PmCh721LXHL92wZ6P0NHG40P7V1vrv7UPre3zb0eu/WQ42m15UJtc3tHGwIb6X8fsV25j+wnNc+NpJBVV0rDy/fakd4wgaSDLJcR5eXtbuMkULVjW38c0MVf16VeL/1+l21tHZEnsb5W39e12uZ0y8g7zzcxAsf7AtbVlZsfrzE/AeWM/+Bt02Xz+tx4fjkO19ld43RdeT1+li3K/ZornR5f3ctNU29Z2q98tFVnP/A28HXnlq7h9uf/6hXuZ3V2TWEeO3OGr6+9D/Utcj1jkRJMshyBxuMsfPDy4sA405jgJ9+7viY77vq9Am9lk255SVe+eiAqc/1AQcbWrn9+Q/p7LLmBjSfz9frbDXpdZleGPjs7r/X7DC62KoTaBkU5PU+lNbtOsI1j69h4s0vJnT9xkqfeWgFX4zQSty0t56apnam3/EK4296ge//fUOEd1szyiqT7nt5M699fJBPDmSuK7KvkGSQ5WoajTO7YWXFANz3ymYA5hwzJOp7vnHGRG44b3LE1773t/dNf/bNz2zi9yt28PYn1ow0mv/Ack75yWuMv+kF7l72cVLraOvs4qVN+yO+FqtaC630kpm/KT+v9wX2655833RyTQevvwvwk4PdXVQ1Te1Umeg2C3CRXUNlW/ytW7nWkTi5zyDLNfr78itKCky/5+YLKqN2CXkTOIi6vEaLwKqzx80Huiut37y1lZsWTDH1vhc+2MeiJ3p3ZfUUfV6i7pZBoGsnUbFGW805ZghHD+3P0nd3JLXuZLWGPOf5ol++Q3VjG/vqwlteZ08ZxqaqOg7UR24FWTiILC6fz8e+ulaOKi9O+X4NSQaJM5UMlFLDgLXAuUAnsBTjRGsTsEhr7VVKLQEW+l+/Xmu9Wik1yWxZS79VDrnzBaOft7wksbwe7SDvMnkxwKprBn9fs5sfPPVB0u//ymOreWvzobjlCvJcXHLyaB5fuTPi64GvM+eeN5KKo+fm+Os3TmXWxMHBf++uaWbpuzsYNaAkqfVHs+1QI8UFeRFfC+03jzQdybjBpTz61Zm8tfkQX3ks8iGYyWTw+xU7+PG/PuLq2RNYfOHUlNZ1pLmD+tYOyosjnyRt3FOHDx/TRg9I6XMSsfjZjby/u5YLp43k2rlHRy23dmcN//fmNuapoVx56riMxRe3BlFKFQAPA4G25f3AYq31m0qp3wAXK6V2AnOBWcAY4GlgZoJlRRICJ0BqeDkvbuzuHnHFOYqjNf/NnlBZ1RpIJRFs2ltnKhEAvPLfc3l7S/TuLJ+v90Xeo8qLI5atqm2Je+YZmggAxgwq5ZTxg3Cn2DHb5fVxqKENt8sYCXaWf9jsSSNK0H/ZSbP/Wc5TjioLG8J7wpgBbOhxF/bM8YNSC8Zi++qMKqaqLnY31uYDDdQ0tXNqj20M3ScpgZbijrsXRlxH4D6XaK9H0tjWyb3/1r2el+12u7h6zgSOHto/5vv/tmYP7Z1evF56JYM7/vUR00ZXcPGJo3hx435e9RxgV02Ts5IBcC/wG+BH/n+fDAQGbi8DzgM08LLW2gfsUkrlK6WGJlJWa23uqBZBrR1dFOS5OH3SYPoVRT47jCZarkikm8hOH++v58Jfmr9xLd4Zblunl8t++17w3+dOHc6eI70rpZqmdk67+3UuriznwWNNfzxgjDhKtvtixZZqbn/+w7CutFDr94XH2vNejpKC3lkoOElhhPVdPmssT6zaRf+izPQkH6hvZcNuo/XyYVU9dc0dVJRGPqv/4sMrOdLckVBFblZHl5emtk4GlBb2eu0/O2pY+u4OBvcrDBswsL++laH9C/neeQowjsuP9zewatthvD6je7LL6wtOkvjRvnqWPLeJ4oI83ttcxYQNrTz7fhUA08cODHbhZnpm4Ji/tFLqq8AhrfW/lVKBZODyV+QADUAFUA4cDnlrYHkiZWMmA6/Xi8fjifuF4mltbbVkPVZJJZ7vPL+HrTXtzBxVQvWhg2Gvbd2yJer7PB5P8P6Cnrq8PlMxbdu2ncZGo2LavXsPHl/vm9zircPs64F49jd0sHxHE/saOqhrjXzNI5otW7ayf1/0M85/rOy+YP3tWYPZuL+Rxub2sBj31LWztca4YP/erqZerwXcOGdYxO/W2tJMa2dy+/HSdw4GE8HIsnw+d+wA/yM8XYwbUMD4MsgrLKIoz0We28WCP2wLe39zc+9rIYeP1OLxeNi1t/drJw3s4Kk8F0dqapLePxPZty98fBtd/l1y5+Fmzrv/dZZ+fmzEskf802T0XHdbp5f3e7R+vvTQGzS0e+n0V8be53aHJcYf/mkFJQUuunzGvv+XD7rfP3FgIeMGFjKsXz5uF7yzswmAu88dxuiK7mTxhb/s4F29l6a6Gl7e0sDuusjDWt0uOOGoEjbsb+HptbtpbDcq+w37q4JlnnzrAw5WG9dvGlvaMlpXxUv7VwE+pdQ5wInA48CwkNfLgFqg3v93z+XeBMrG5Ha7qaysjFcsLo/HY8l6zDrc2MYTq3Zx2SljGVpWZGk8W/0H/KAB5Vx2RiVbGz3B0SuTJk0CIt9vUFlZ6R8Our3Xaz7gSEceZ51kxGRUONt6lZswYQL9dRvQwpgxo6mcMjzk1W3Bz4ks9PXe6w6NE7q30YKbXohaNp5Jk46mylcNRO4qqvUav80Ht51HeXEB1z+5nh0NR5h4zGSK8vNo7/SyYPGyYPkDTV182FxGQZ6Ltg4vM44eCBh3Hl97wYyI3XTlKxvwtnTE/L27vD7u+NdHfO308Ywb3C+4vGh9C2Akg4nDB/D9z8wKe1/v/Sh8u5aWlgLhF4/79S+jsrKSQ3mHAKOL0eUyulrmnXws7tcOMHjw4KT3z3j79hOrdpHnhktnjKHLFx7vgcbOiO8N3R9rCoZy2tGDg9v6R89s7FV+5e5mThk/iKICN20tzVQ1+fAc6j4p+Num8Konz+2itCCPOZOHsKO6mRU7G+nyn9mDce1p1olTw65FjBlczZq99azZa6x3xriBLDh+BNPHDqByRDkuF7hdLtwuI1H7fD5cLhd1zR386Y31XD7vRHzA9Dte4d19XRxV3h9opMPnpv/wcfQvymdgv94tFbPWru05e0BkMZOB1vqMwN9KqTeBa4GfKaXmaa3fBBYAbwBbgHuUUvcCowG31rpaKbXebNnEvl72eHrdHu57ZTNut4tFZ06yZJ3GePzu0R+F+XmMH9KPR748g/H+CjNet0isawpXPbObHadMA2I/JjLdHUrtnV4K842zuJueTv7aAsQfIlnb3EG+2xU8yPPz3OyuaUEtfokddy/khY1Vvd4TOjb/9Rvmdn9WlG2b53ax9WDsO5E37a1j6bs7ePb9vbx/63nB5a0d3edVRfnWjAjvjPAsi7WLz2VQSMWTzt/45n8YlfeNT/euxAFufW4TOw83s37XEepbe98BH7hrviDPRUdX5Ejv/MxxwX73QHJqbu/EhYvCfDcdXd5gJe12xT4u6po7aGzv7HVR+slvnEpVbQtlxfkM7ldESWHsLtvAZ1SUFnDWxDIG9isMTgq4fld3cqpr6WDOPW9QmO9m/S3n0i/NXXbJrP0G4BGlVCHgAZ7SWncppd4GVmLcu7AoibJ9UuAgjjaUMxkPvvYJD7z6SfDfM8cPTHgdZgeJ+NJ4q3G8dU9evIzTjh5MKW28ujW90zl0dHnD+oFD714ef9MLjB5ojAIaWFrApTPHUFtTw183dh+4ZrbSkeZ2Gts66fL6wu5YPljfSlVdKx9V1QcryIbWTuqaO3hz80E27a3jVU/3/QrRRg/F8rNLTggbKVVc4ObCaSPCypw+aXBYInDhSuvvH+rz00dT19LBF2aMDs6D9fjKnYwaUMKYQaV8WFUf9b3REkE0pYXd1V6e2/y2rCgtiHgdo6KkIKGh3ZG4euT3X18xnYbWDtwuF6MGlFAaJ8FYwXQy0FrPC/nn3Aiv3wbc1mPZZrNlhXkfhEz//OsrpnPB8SN6lUn1ZqFAhRXremcmRh2+u7X78tKSi6by7tbDSd3I5XLB5GHRR3u0d3nDbhxr6ZG8C/PdXDpjNP/7+Wm4XC7eW78pLBms2hZ/YsCz1DDW76qlo8tLnjuPj/fX8+Crn7Aswk1yXV4fJ/z45V7Lxw8uZXaMGwqjGTOolDs+cxy3PLuJiUP68fr35wVfi7avBLqMEuH1+nhuw14K8twM7erC5/NR39LJcxv2cqC+lU176zlQ34rbf3Z8xuShPPqVGWGJ+OITR/Lc+1X87JJpfGHGGMBI1np/Q9RBA89/ZzatnV18IYkbBp3AHdIi8fz4/Liti3SQm86yUIcF0z/E60ZqaO1gQGlh9GcGW9CB8Mf3Io/5j2bc4NKEZ2MNNWviYL5xxkR+u7z3dYqOTl9YhfT3az/F6u01/OzfGoAXvjsn7ADtVxh+KtfeGb/lF3j/ul1HuPJ3q4KJtnJEOZ87aRSThvdHDS+jvrWD1dtraO/08vNXNpPndvHQFdOZOX5QUq2CgESTt4vEu4keW7GdO1/ovuh5c0MJTW1dPPia0ZItK85n+tiBFOS5GTmghGvnTow4lUdPBXluBoSclV86YzQtHV6e32B03x0/uoKmtk4qR5Tj2dfdisjkfRKpCL3Hzo5EAJIMslJnSLM42X093n0ITe1dDChNcuUm2TFfz5hBkb9Ue5c3OA01GGPwZ44fFEwGhT366fPdLh758gyueXwNgKkLfPn+I/6dT6qDieD+S0/gc9NHh5UbSQlTjioH4Oo5E018K3MCP7nZCt7lciXcMgjc4Xz17An87p3t/H7FDtwuFwV5LjbfuSDuftfz86O555ITAILJAKBfUT7LrpsTvG6WTdwOyFoyN1EWMtMySHXfOvd+4/aQf27ofeEUMnMBOZLTJyXeRWJGe6c3bBrqnnrOSgrGvQi/+/IMwFx3SoE/obzunyL61e/N7ZUI0inRrkMXsOVQI9urm2jt6Ip4/aCjy8srHx1g6YrtPLJ8G4++sx23CxZfOJUvnziQoWVFDO5fyDfPODqhRGCVbJlbyQnJQFoGWagjpCM/XftQc3sXR5ra+WEKdwinwxWzxnLescM55SeJPSQm3nbaW9vC+MGJN4UC+SMw9HDa6IqoZQv8hQM3hA2IclNVuiS6rzS0dbJ88yHOvPfN4LJRA0ooK85nb20L+W5XcMx/qLP8w4y/dMJAfpzCMO6e4dqRTDLFCY/OzulksOVgAwV57rDx3NlA748+siLAin3rpDteSftnJMrlcgVnaLXa9HG9R2UtPH4E62M8iyBQQd3gH2Z6TYxunZ51WabPBgOfFm2EkJnWTeiDgr50yhhKC/MZO6iU8487itLCPPLdbtv6vLOZtAxs1OX1cc79y4HE5iexk97fwDWPrwkbc25PlWzIjokrzDn/2KO4/9ITey3/5ZdOosMbvVuurcP8xfyeD9fpjLHedEi0vlmz+By++ce1XDNnItf+yRjuuXhhJeUlBVzqH+WTTj3jjRT+8aMqGDkg+smBA+pYU5wQZ84mg0zP+2GFB1/bzK6aZory3bTFi9+CneuzJ43iH+sjP+Dd6U866yleF0O0ywVut4uiGGPRE5ny+typR/GlU8bwl9W7ASjKy+wZdKD/3OxPN6R/EU9/67SwZVZe0LbC0986LeL1nGwT2D8XRhgmnik5ewE51tmeE63ZUcOLG/dzxuShLP/hmcHlqZxRnDQ29vS9848dHvU1O56Alc6LgXlJTifaczvE+j3y3C5++rlpfHzH+fz7+jOiTsSWNllWZ/ZqGUSIvzDf3SeSAcD6W87lgct6t04zJWdbBqHDMwNzhTjZtX8ypuSdNcHctMNWVJzJVpBOFG9rRHhQWdoUF+ShjiqLX9Bi8b6iww+BpGTTV0pl/iEr9J2jPY6DDa3c9s8P2VZjzOkT2l+bDU9Fqm1u5/JZY1l05qSwHTydO3u8e4EyfaClszVi1QU8Jw9lDJzwZEsXX89t6eRt2xfkTMvgnU+qWfruDt4dXETFsGr6h1zM6/T6yE+x+/b1jw9w3MgKhkV5IEoqurw+Or2+qA9bSZdYFaRd9xkkK15dn+xjFrOlYoXsOksGe7oic1nOJIPA2f/mw21c+egqRlR0V6wdXd6ot/m/t+0wNU3tYWX3HGkJu/HrlY8O8GFVPaMHlvDOjWeFvT9QWaRSaTS3GzM2JjJZVbInup+aOJiV24z5gJww3C1UWq8ZJPld03UTXDo47OdMWDLxZ/t3zqScSQaBOXZ+OGcYqw/6eDPkwWoX/OJt8lyusPMQnw/qWzuojXBTTTSDI/T5xXoIu1lP+kefBKewDdnBrb7WETpZW6wLc8l+KyvmVUpGvESS7OWR40ZVsGHJeZxwuzGpnJMrn+7pKOSMW/SWQ8nA+P/xw4sprugfTAYTh/bj+FHdd40GjuVAJesCrjh1XNjj/wb1K2RI//CK/9KHV0aczsCKw27PEWP44nlTo4/u6SnZOqkw5EKB2ZZBImfs/0lhorl09nOk0grqlyU3WcX7nZze5eXgPNsn5EwyCHQTuV1w1ewJXHnqOFwuTM2YaEa+2x3xQnTgbs9UWggdXh9D+hcyuL/xNK7Qg9rqAyR0QraYLYOQ75PImaZdo7bifewlJyc/R1C+RftQugVbBg6v9K0kF53Ny4692AKByitwBliY77YsEYDRvRLpjlJvMBnEfv++uhbuetHDii29H/rW1eVLeCx1spVu6DZJdfN4vT7G3/QCn/5V9xz0T6yO/ChOu500NvEHBIUaXh5I1M6VbaOJenHyxu0DciYZdEV4xJ+V8tyuiC0Db/ACsvHHii3VzP/5cg41tNHY1kl1Uyd6fwPzf76c3y7fxu3P957WudPrC+uCCv0OVn+f0GsGsbpO3ovyMJeW9u55/Sfe/CLQ/TCew41tvL87+jw/2SwbzkCdH6GwU850EwUq5XTdrJjvjvwc1sBZ2O4jzdz3suaXr28B4FM/fS34DNrQB9cXRRjj2uUNfwpXOpWFXBsZ4u+WiuR/X/qYeWpor+WVt74Usbzdc8yne+s5+cJxQDbEGEtSCTfLv3Mm5VAyCO8mslq0lkGVf5bHFzeGP9qw0+ujX2EeV0yroLWgjDEDS1m2aV+vB383t3fS2uEN6yYKu+ksytdJ5aE3FcVu6lrNJ6DAs3ID8/TnMidXuNnQehH2yblkkK6DNT/PHfGawf761rB/77h7IQ2tHZQVG/PSeDweKv1zvnv21/PMur1sPtBAZ5ePP763Izip2eTh0Z/fazVfsBUVe2N9csB4SL0PH4+v3MmSf6b3yWUp/XRprgezoZqNt+87OZGB8+PLdjmUDIz/p7ObaMfhZsbf9AInjBlAZ5eXvbUt1DZ3UDminD98bSaD/PchBBJBT5+aOJhn1u3lvJ8vDz4UPCD0xrfQi8OxHmaeKjMPhAG4auma1D8sRUsumsrtz39kdxg4OS3Ee55BX+TcX8N5cu4CcrqSweWnjGXeZKMPfcPuWj6sqg/esDZqQAnDyovjDkGcf9xRwb/31Ya3KKob23sWTxuvyZaBk8QbGZbuLhKnT3QI2X9mneXhO17OtAx6Di212qyJg5k1cTC/X7GdGeMGMWlYf9btOsKHVXWcNWWYqXUUh1w8Xr3DGK1z64VTeWnTfmaM7x76GPYNol4zSP17ZtPBZ/pnzaGzYoCvnT6eUQNK/P/K7pvORHrlTDIIzIKQ7grua6dPCP59+qQhCc1dUxDhgu1Vsydw1ewJEUqnX2jizHe7QkY/OU+85Jeps2KnnX0vuejY4N/d01Fkp2RaX9nQYnOKnOkm6h5NZHMgMYTuuIV5bn4wX0UpF/J31JUlH0ekkVefn578HbqOkqbKIRvqnCwIUdgoZ1oGXp8Pl8v5Zwpb77oAF8lPqWyF4JljSAijBpaElcl3u1h7y7nBCdrsFnekjM2f7wRO3/fjye7onS+nWgbJTlOcSXluV9xEEDY3kcXfyeUKHVravfzb844OK3flqePIz3DCivVdnfLLOiWOSLpHE9kahmm5dt3LbjmTDLq82TU6JlXJflWfr3s0UegUGD1HQjntubN2/7TZcEOX3dsoUYk8X1qkLmeSgc/nS3rOesdJ40Hh8/mC1wx6Vvih01Pku13BBNwRAAASg0lEQVSOOjjjX0DOTLBO7opxcGjCAfpK9RhXW6c9D1VJt2jHdyrH/Wlj+wG9k8E/vn1a8O/jR1c462zY5qGl2VDRBn6vbH24jaP2tz4oJ5LB3toWlr67gwizRWSldFY8breLG2YPY9XNZ/dKBmMGlQb/vnDayPQFkYR4myTtF5DTvH5LZPl0FCK9ciIZ7KkxnhT2ldPG2RyJ9aJOVJfkke3CRWGei+HlxfHLZmPlkeagnbxJXMH/R47S8ReWnbxx+4CcGFra0mHMsb/g+BHQtD9OaedL5zFhdCE486iLFVW85JfuxOXkawUBk4eXcfyoCi6dOcbuUIQDxU0GSqk84BFAAV3A1zCOy6UYQ9I3AYu01l6l1BJgIdAJXK+1Xq2UmmS2rMXfLSjwwJXSwjx8Ten6FHvkUj/q+MH9or5m91aw+/PNGDmghOe/O9vuMIRDmekmughAa306cCtwv/+/xVrrORjHwcVKqenAXGAWcBnwkP/9iZRNi7oWY8K4koLseHB5PGbOQpOtnJzcVTCkrDDqaz03yUUnhF/TyFTSzIIGQtaSbZtecZOB1vpZ4Bv+f44DDgAnA2/5ly0DzgFmAy9rrX1a611AvlJqaIJl0+KmZzYC0L+o7/WK2XmAOOng7BnLL790Eh/9eH4GA8jcR5mhhpfZHYLIMqZqR611p1LqD8BngUuAC7XWgXPIBqACKAcOh7wtsNyVQNlDSX6PqLYeMh7AMnP8QAb3L6IvPIvLTL2TbEXttJvJzErlzH/RmUfHL2Q2Dodsvue+czptHX1k+JyfQzZtn2X6VFlr/RWl1I3AKiB0opoyoBao9//dc7k3gbJReb1ePB6P2XCDfr2qGoD54wvxeDy0trYmtZ50SSaelpCDfNeuXQzq6J1D9cc66vsDn9fS0hK2fPrIEk4f1klra1evmCL9O9Izn9NJf6wpLojcmK2qqgr7t8fjoTVkO23+RNO/MI/9++p7vfdTQzoS/g16/m4d7cbzJnbv3oPHV5PQuqwQbT+qilC2p11Vxmi7pqamuNshke2U7LFWV2f8RlV7q/AUNwSXN7V3/55m11u1rwqPpyGleNLJSTGZuYD8X8BorfVPgWaMyn2NUmqe1vpNYAHwBrAFuEcpdS8wGnBrrauVUuvNlo0Vh9vtDj4eMhEPKh+3NLUxrMwYKhn6mEknSCae5vZOYAcA48aOpTJsmuxtAEyZMiVYpqfA55W8UQO0BZc/8//OihDTtrD3hP67o8sLbE8o9lRMmTKFksLAdZ9tYa+NGjUSQtp9lZWV/oEDOwCYPFlRUVLAmrqdQPiudswxx5gaShuq5+9WtOwg0MGYMaOpnDI8oXVZIZX9+nB+NbCffv36xVhHz/0gfTFVbGgFGhk5aiSVld2z5Ta2de/3sdfbvW+MHNG9Dqcd+5CZmNauXWuqnJmWwTPA75VSy4EC4HrAAzyilCr0//2U1rpLKfU2sBLjWsQi//tvSKCs5dxuVzAR9EnSdjYlVveNFZsw3hh+IZwubjLQWjcBl0Z4aW6EsrcBt/VYttlsWWGOmQonE33XTqr27B7n75RrBcnIltizJMyslRN3IPdlciZqiLQVTD0EyLLPz97fwcnDiVPVh7+a5SQZZCGnnMnZfTYeyjGhOCWOPsgxv3EfJckgy+XqATLnmPBnS6c0hbUF2zBXfwenk5/FPEkGIml2HmizJgyybmXSl+Ao0adlT3yPk5/WPEkGfVRGLiBnOBvEPLm3+RnImf4cIawmySDLSeVjSGk7WNJNJL9EuiWzieVXMU+SQRYyc1BkYnSLkypAp4TipG0ipJsoEZIMspxUPgG9t0PY0NJ0P88gvasXIu0kGWQhp9x0lqwHLzvR8nUW5stNZ8nK5tjj6cNfzXKSDLJcNh7I6bjJqSg/9rMqMvY8g4x8Sm5KZl+XbiLzJBlkoWxMAHYwmwCsSBTym1hHKnB7SDLIctHHZDuXLw2He7zvm/5rBk7e4n1DMttYfhXzJBlkoWzfwdMyF45DNoq0EES2kmSQ5aJVPk4eZTRpWH/L1xnprDGTm8DBmzuubJmoLpu3cTaQZJCFnFzRmzFt9AC7Q0gb6S5ylizJc47Q954Qn3MiVz65ViXFnY5C7jNIyRNXz6LL5iZEX9/GdpNkkIXkoHCgLG+txXPapCHxCzlQ3/5VrCXdRFmuj9dBKTG7aazchtn4e2RLzMl0j0o3kXmSDLKQqbmJsuQAt0rcoaVpPkfMsc2dVrIt7SHJIMvJgWOw+6J6riVfOySzieVnMU+SQRYyU/HZVTmq4WUAHJOG4aOxRPq6odsgU5tDKh9nkW4i8yQZiLQ4dmR5Rj/P7krY7s/PBdL6Si9JBlnO7u6RaJwWV8aicdbXznnyc5gnyUD0CZG6AzJZETgt+QmDdBOZJ8kgy0kVZIg/UZ1Roji/9y5vxTbM5t8he6ajyOat7HySDERaZPqwNVtPfPakUfxgvgpbZmVdKNNRiGwlySDLycmSOYHNlJ/nZtGZk5g5fiAAX589gcH9ClNffxb/Dtkcu7COTEchUvL12RM4Uw3jykdX2RxJYrOW/uWaU+n0+iguiP2ENPOfLjWqyG6SDLKc3ZXQLRdOjfyCw+vG/Dw3cZ6UmRiHf18h4pFuohzzf1dMT+n9nzlxpKlydiepnjI1a2k6nuImRCZIMshyiVZyC44fkdLn/fyLJ7L1rgtSWkemZHL0ifS7i2wn3UQ5YPTAEr4172hLhhC6XC7yHFjxxX+eQYaCloaByFKSDHLAOzeelfHPzPSZst35yWndYkIkSpJBH3b5rLEsNNktVBjhZqy+4PZPH8vQsqKMfV42NgwK8ozfvn+RM6qDbNyGfUHMX18pVQA8BowHioA7gY+ApRi/2SZgkdbaq5RaAiwEOoHrtdarlVKTzJa1/qvlhlhn4Hd99viY7731wqn8zz82sfSqmfQrdEZFkKxo3UBfOW18hj7f+H+23M0baub4gfxowRQunTHG7lCEjeKdDl4JHNZazwEWAL8C7gcW+5e5gIuVUtOBucAs4DLgIf/7EykrLPDOjWey8kfmuoVOGjuQF6+bw7CyYvo54Kxw9c1n89Dl0yO2UipHlFOYF313nTois7Ok9pTNF5BdLhffnHs0Ay24+c4K8TblmWpoRuLINfFqgL8DT4X8uxM4GXjL/+9lwHmABl7WWvuAXUqpfKXU0ETKaq0PWfKNckzPvurRA0ttiiRconXjv747m2HlxSycNoI39EGeWrsn+Fr/QjfLrpsTVj7wPe/8zHFcOG0EhfluNt+5gFue3cQ1Z0xINfykydDS9PrP/5xDWbH9Jy59UcytqrVuBFBKlWEkhcXAvf6KHKABqADKgcMhbw0sdyVQNmYy8Hq9eDweE18pttbWVkvWY5VU49m+fRuuOmv7xCPFZDbGtrZWAOrq6qKW6bmuY4cVk1dfhae+yv/e2rDXfT5fr/dMLvJx13kjOLG8iX07t7LPv/yrxxbQcXgPnsOkVc9t1NzUDMDOnbsY0lmd3g83EY8TJBtTXX09AFV7q/AUN/R6Pd7WnXZUMR/sN/bDqqoqPKUNKcWTTk6KKW6KVUqNAf4B/Fpr/YRS6p6Ql8uAWqDe/3fP5d4EysbkdruprKyMVywuj8djyXqskmw8V53uY8+RZs6ceRylFvf3h8e0DcBEjEa5oqJioIMBAyow8nxvxrq2Bf/dr7Q0bP0DNrUDjcF/VxTnRfz8qVFufs6Enr9b/3frgRbGjh1L5eTMd2M4bb+G5GP6dslwtv5tA58+7TjGDEq8pfvExGP46mOrWbPzCCNHjqSycnRK8aRTJmJau3atqXIxrxkopYYDLwM3aq0f8y9er5Sa5/97AfA2sAKYr5RyK6XGAm6tdXWCZUUCbr1oKr/98gzLE4Edjh9dEfbv8487Kvj3zRdM4afnmbvr2Qmkkyh1M8cPYvkPz0wqEYAxKmpsku/NZfFqkpuBgcAtSqlb/MuuA36hlCoEPMBTWusupdTbwEqMBLPIX/YG4BGTZUUfYnbc/fPfmc2UEWVhy86uHI7nx+dTlO/G7XY5phktskgWX9C3S7xrBtdhVP49zY1Q9jbgth7LNpstK5zpns9PS+tF0Z6tgoCSQitnkcscXzaOLe2L5GdIWPb3MYi0unRmesaeL7tuTtbf2xAqcJ+D1EEiW/Wdo1E4Srxx95U23xdgNemVcBj5QRLWN+cgEMIu0jRwBvkdEibJQKRFNt+Rm4zqxjYAWjq6bI5EiORIMhBpseC4ERxVXkyeOzeywuxjhgAweXh/myMRgHQTJUGSgUiLYeVFvHfz2Uwc0s/uUDLipvOn8OHt85k0rCx+YZF+0k2UMEkGIq1+dMEUKkoKgv/+73Mm2xhN+rhcLkdM9idEsiQZCEt858xJFBe4+d9LpnHa0YOZOMToLjlrynA2LDkvWO66c46xK0SRS6SbKGFyKiMs8f35iu/PVwA8cc2pNkcjhEiUtAyEEEJIMhBCCCHJQAghBJIMhBBCIMlACCEEkgyEEEIgyUAIIQSSDESGhN6FLIRwHrnpTGTEqpvPtjsEIUQMkgxERhQXZOdjLIXIFdJNJIQQQpKBEEIISQZCCCGQZCCE6IP6+58tkSc1nGlyAVkI0edcM2ciXV4fZ00ZbncoWUOSgRCizxkzqJSffPZ4u8PIKtKIEkIIIclACCGEJAMhhBBIMhBCCIEkAyGEEEgyEEIIgSQDIYQQSDIQQggBuHw+n90xmLJ27dpDwE674xBCiCwz7uSTTx4ar1DWJAMhhBDpI91EQgghJBkIIYSQZCCEEAJJBkIIIZBkIIQQgj7yPAOlVAHwGDAeKALuBD4ClgI+YBOwSGvtVUotARYCncD1WuvVSqkTgV8CXUAb8GWt9QG74glZz+XAd7XWn0o2FqtiUkoNAx4BBgJ5GNtoq43xnAj8xr9sM3C11tqbbDyJxuQvPwl4Vmt9nP/fQ4AngBKgCvia1rrZxnjG+t+fD7iAb2itdbLxWBFTyHrOAP6stR5jZzxKqX7A/wETgEKM4201KbDod/sjxm9WA1yeyn5kVl9pGVwJHNZazwEWAL8C7gcW+5e5gIuVUtOBucAs4DLgIf/7H8TYCeYBzwA32hwP/sru6/6yVkg1pnswDt4zgMXAFJvjWQL8WGs9G+OAW5hiPKZjAlBK/RfwJDAk5P23Ak/4y64HvmlzPHcAv/Lv13cBP00xHitiQik1BrgBKHBAPD8ANvnLXgMoB8T038Bf/cfahxj1QNr1lWTwd+CWkH93AicDb/n/vQw4B5gNvKy19mmtdwH5SqmhwGVa6/f9ZfOBVjvjUUoNBu4Grk8xDstiAk4HRiulXgWuAN60OZ71wCCllAsoAzpSjCeRmACOYCSpULOBlyKUtSueG4AX/H9bsV+nHJNSqhijRfdtC2JJOR5gPtCulPq3fz3/dkBM72O0wAHKsWbfjqtPJAOtdaPWukEpVQY8hXHm6tJaB+6oawAqMDZsXchbG4AKrfU+AKXUacB3gJ/bGM8g4FGMs4OGVOKwMKYKjCbvEa31OcAuUmw9WRDPJ8AvAA8wnNSTUyIxobX+l9a6qccqQmMNlrUrHq11tda6QymlgHuB21OJx4qYMM6S79Va7001FoviGQIM1FrPB57H2E52x7QH+I5S6kOMlsXfU43JjD6RDCDY9HwD+KPW+gkgtP+4DKgF6v1/91yOUuqLGGcsC7XWh2yMpwI4BqMf80lgqlLqgVTjSTGmWuAw8E//sueBGTbH8yAwR2s9BXgcuC/VeBKIKZrQWOOVzUQ8KKXOBJ4F/ivV6wWpxqSUGgnMAZYopd7EaNk9aVc8fpbv1xbE9DPgq1rrY4HrMPbvtOsTyUApNRx4GbhRa/2Yf/F6pdQ8/98LgLeBFcB8pZTbf5HGrbWuVkpdidEimKe13mZzPKu11sf6+3kvAz7SWqfcXZTqNgLeAS7wlw30ZdoZTw1G5QvGxdqBpCiBmKJZQfc2ilc27fH4E8GDwPla6zWpxGJFTFrrKq210lrP8+/fNVrry+yKx8/S/dqimI7Q3cK0ZN82o0+MJgJuxthgtyilAn111wG/UEoVYnQlPKW17lJKvQ2sxEiEi5RSeRjdDbuAZ4wWNW9prZfYEU8Kn5numG4AfqeU+hbGjnq5zfFcDTyplOoE2jEu/qXKVEwx3n8n8Ael1DVANRnaRjHe/wDGCJk/+PdrrbVO9aJ2qjFZLdV47sLYr1di9M1/2QExfRf4lb9ucpHeeiFIJqoTQgjRN7qJhBBCpEaSgRBCCEkGQgghJBkIIYRAkoEQQggkGQghhECSgRBCCCQZCCGEAP4/scaiqIRpsuwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "daily_df['size'].plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the cumulative number of revisions by day."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a249fef60>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD7CAYAAACG50QgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VPWh//H3TPaEJBAIAWTfvqxuQRA30Lrblnq7SK1Fa+tW2mqv916f9qfV6/V203pbr9u9qFVrbe+V1rr0Wq0LiIpQIorI8AXCEgiQkH1fZub8/piRRgxkm+ScyXxez5PnmTn5zpnPJJNPznznzDk+x3EQEZHE4Hc7gIiIDByVvohIAlHpi4gkEJW+iEgCUemLiCQQlb6ISAJR6YuIJBCVvohIAlHpi4gkkGS3AxzpvffeczIyMvq8ntbWVtLS0mKQKHa8lslrecB7mbyWB7yXyWt5wHuZBiJPU1NTRWFhYX5X4zxX+j6fj5kzZ/Z5PYFAICbriSWvZfJaHvBeJq/lAe9l8loe8F6mgchTVFS0pzvjNL0jIpJAVPoiIglEpS8ikkBU+iIiCUSlLyKSQFT6IiIJRKUvIpJAPLefvohIInAch79sPsi+6mbaQmHaQ2EumTuaaQXZ/Xq/Kn0RkQG0u6KRR9/axYub9lPd1P6J7+Wkp6j0RUTiUXsoTHVTG02tIT482Ex50iHW7azkwVXFAJw5bQSnTh7OlwvHkpORQmqSH7/f1++5VPoiIl1wHIdQ2CHkODS1hqhuaqMtFKYtGGZHeQM1Te2Ewg6NbUG2lzdQXN7A7spGWtrDHdZy4PClp69ZwGlTRgz8A0GlLyIJqqE1iD1YR1ldK6XVzbQGQwAcqG1h7c5KSqubCUfLPux0f725GSkcPzaX48fmMm1kNsOyUmmsKmPO9EmkJiUxMieNgpz0fnpUXVPpi8ig5TgOtqyewIE6SiqbCYbDhMIOe6ub+cvmA7SHOm/zKflZfHX+eDJSk0jy+Ujy//0rJclHfnYaaclJpCb5OW5YBmNyM/D7IcnvIz056VPTNIFAPTMn5A3EQ+6SSl9E4l7RnmqeXLubioZWGlpDNDY20f7iQQ7UtNAW+vsUi98Hfp8Pv9/H9IJsblg8hckjhlCQk0ZORsrhccl+Hz5f/8+vu0GlLyJxKxgKc9efA/x23R58+JhWMIThQ9JIc5LJy83hnBkjGZ+XySkT85icn0VmqipPPwERiTst7SH+vOkAT63bw8aSGi6eO4pbL5nFmKGREzB57Xj6XqLSF5G40dQWZJU9xN0vW3ZVNDI6N507l8xm2cKJbkeLGyp9EfG0cNjhneJKXtp8gOc/2E99S5DUJD93fWEOS08ZR3KSjibTEyp9EfGszaW13PnCFtbvriLZ7+OcGSO59KTjOHN6PkPSVF+9oZ+aiHhGfUs7H5bWsq+qmWc3lrJ2ZyVZqUnc9YU5fP7EMeSkp3S9Ejkmlb6IuCIcdnjxwwN8VFrL9vIGKhta+bC09vAHoVKT/SxbOIEbPzON4UPS3A07iKj0RWTABUNhzv+PN9lZ0QjA5Pws8jJTueyU8XxmxkimjBzCqJx0MlKTXE46+Kj0RaTfNLeFOFTfynsl1dREj1dzqL6V3/9tL/UtQc6cNoLHvzGfpAE40JhEqPRFpFf2VjVRtKeabbtreWHPVpraQrQGQ9Q2t7O/poXtZfU0toU+dTu/DybnD+Gzxw/jziVzVPgDTKUvIp/Q3BZiT1UjB2paCBysY8Puaiob26hrbqeuuZ3m9hDBkHPE4Q0qyU5PITXZT3Z6MsOzUvli4VjyslIZlZNOQW46J4wdSmqyn7RkPynazdI1Kn0RwXEcnt1Yygd7a1hZtO8TW+iTR2Rx3LAMxudlMiQtmSFpSST5/QzLTOGMaSOoK9tL4dxZpCaryOOBSl8kwQVDYW5//iN+u66EJL+PmaOz+fqpE5g6cgjj87LIzz72njOB2v0q/Dii0hdJUGV1LfxufQkvbjrAjvIGvlw4lp998fgBOXuTuEelL5JgivZU8dCqYl7bWo7jwIxR2fz40rlcvmC829FkAKj0RQa5vVVNvL61nN2Vjby1vYLt5Q1kpiZx3swCvn/edGaOznE7ogwglb7IINMeCrP1QD0fltaydmclL28+SFsojM8HheOH8Y/nTecr88YxKte9U/aJe1T6IoNAOOxw9yuWVfYQuysaaW6P7H0zJC2ZpfPHsWzhBCYMz9KuktK90jfGjASKgPOAIPA44ACbgeXW2rAx5nbgkuj3b7LWrjfGTO1sbKwfhEiie2rdHh5aVcwJY3P58ryxnDhuKIUThjFuWKbemJVP6LL0jTEpwH8BzdFF9wK3WmtXGWMeBpYYY/YAi4AFwDjgD8ApnY0Fno39wxBJPO2hMIEDdfyhaB9PrN0DwLPfPl0lL8fUnS39e4CHgR9ErxcCq6OXXwLOByzwirXWAUqMMcnGmPyjjFXpi/RSeyhMVVOQ1wJl/OSlrewobwBg/qQ8fvTZWSp86dIxS98YcxVwyFr7sjHm49L3RcsdoB7IBXKAyg43/Xh5Z2OPKRwOEwgEuv8IjqKlpSUm64klr2XyWh7wXiY38rSFwlQ0hihvDFLfGqIt5LC3tp21JY3sq2uPHnq4hLQkH9fOy2PuqAymDk+Duv0E6vYPaFbw3u8MvJfJS3m62tK/GnCMMecCJwJPAiM7fD8bqAHqopePXB7uZNkx+f3+mJzQ2IsnRvZaJq/lAe9lGqg8FQ2t/OrV7by7s5Lt0a33jnw+mD0mh2+fPIFwUzVzp07glEl5jPDAcea99jsD72UaiDxFRUXdGnfM0rfWnvXxZWPMKuB64G5jzGJr7SrgIuANYAfwc2PMPcBYwG+trTDGbOxkrIgAW/bX8bO/bD18tqj2kMPCycO5aO5oxg3LYFRuOiOGpJGekkR+dtrh0wNGCmS0y+klXvVml82bgRXGmFQgAKy01oaMMWuAtYAfWH60sTHILBJ3tpXV8/DqYt7bU01ze4jmthB1LUEyUpIonDCMxWYk1y+aQuGEYW5HlUGu26VvrV3c4eqiTr5/B3DHEcu2dTZWJFHsq27i8hXrKKlqAuDEcUM5dVQ2acl+cjJS+PwJY5hWkN3FWkRiRx/OEukn4bDD9363kZKqJr73mWlcMnc0ZpQKXtyl0hfpBzvK6/nnlZvYWFLDHZ+bxVWnT3I7kgig0heJOcdxWPrf62hsDXLPl0/giycf53YkkcN0IA6RGLNl9VQ0tPKFk8bwpcKx+Hz6wJR4h0pfJMYCB+oAuFpTOuJBKn2RGNt6sJ6UJB8TR2S5HUXkUzSnLxIDwVCYd4orWb+rikfW7GLayCE6jLF4kkpfpA8qGlpZ8eZOnv9gPwdqWw4fLuGHF3nnEAAiHan0RXopHHY4797VVDe1R46Lc/ZUvnjycWSm6s9KvEvPTpFeen1rOdVN7Xz/3OnceO40t+OIdIsmHUV66en1JWSnJ3PD4iluRxHpNpW+SC/sPNTA61vLuX7RFFKT9Wck8UPPVpFeWGUPAfC548e4nESkZ1T6Ir3w+tZyJo3IYvzwTLejiPSISl+kh+zBet7aUcH5swrcjiLSYyp9kR5avytyOuivLZjgchKRnlPpi/TQlgN1DMtMYVxehttRRHpMpS/SQ4fq2xiVm6GjZ0pcUumL9FB1UxvDMlPcjiHSKyp9kR4qqWpi7DBN7Uh8UumL9IDjOFQ0tFKQk+52FJFeUemL9EBbKIzjQHpKkttRRHpFpS/SA63BMABpOvSCxCk9c0V6oLU9Wvra0pc4pdIX6YGdhxoAyFDpS5xS6Yt007qdlXx1xbukJfs5fepwt+OI9IpKX6QbHMfhZ3/ZSkZKEi9+9wxG52qXTYlPKn2Rbnjq3T28V1LD98+bzrSCbLfjiPSaSl+kC5v21XDbcx+R7PfpIGsS93SOXJGjKK1p5nsv7mN75U4AVlw5j4xUvYEr8U2lL9KJ2uZ2blm5ie2VbVx31mSuPG0iY4ZqHl/in0pf5AjldS1c+uA7lNY0c9LoDH5w8Uy3I4nEjEpfBKhvaefNbRVsKq3hkTW7CIUdHr6ikDFUuR1NJKZU+pLwHn1rFw+8sYOqxjaS/T7GDctg+dlTuXDOKAKBarfjicRUl6VvjEkCVgAGCAHfAHzA44ADbAaWW2vDxpjbgUuAIHCTtXa9MWZqZ2Nj/1BEem5zaS3/9uIWxuVl8C8XzOXSk48jLVlv1srg1Z1dNj8HYK09HfgRcG/061Zr7ZlE/gEsMcacDCwCFgBLgQeit//U2Jg+ApFeag+FuerXfwPgyasXsHT+eBW+DHpdlr619k/AtdGrE4AyoBBYHV32EnAucAbwirXWsdaWAMnGmPyjjBVx3Z82llLR0Mq3F09h0ogst+OIDIhufTjLWhs0xjwB/CewEvBZa53ot+uBXCAHqO1ws4+XdzZWxFX7qpv4wR8/ZOLwTL73mWluxxEZMN1+I9dae6Ux5hZgHdBxh+VsoAaoi14+cnm4k2VHFQ6HCQQC3Y11VC0tLTFZTyx5LZPX8kD/Zwo7DlvKW3lofQVhx+FHi4aza8c21/L0htcyeS0PeC+Tl/J0543crwNjrbU/AZqIlPgGY8xia+0q4CLgDWAH8HNjzD3AWMBvra0wxmzsZOxR+f1+Zs7s+37RgUAgJuuJJa9l8loe6N9Mq2w533l6Iw2tQbLTkrnts7M4Z/4k1/L0ltcyeS0PeC/TQOQpKirq1rjubOn/Efi1MeZNIAW4CQgAK4wxqdHLK621IWPMGmAtkWmj5dHb33zk2J48EJFY2FZWzz898wHpKX7+8bxZXHrScQzLSnU7lsiA67L0rbWNwFc6+daiTsbeAdxxxLJtnY0VGSgbdlex7LH1tAbD3P/Vk7ho7mi3I4m4Rh/OkkGttqmdm/7nfUJhh7/ceKYOiywJT6Uvg1Ztczvn/GIVlY1tPHn1fBW+CDqevgxSobDDNU9soLqpjeVnT+Gs6fluRxLxBG3py6BT0dDKt57YwPt7a7jlwhncsHiK25FEPEOlL4POo2/t4v29Ndy5ZDZfP1VnuhLpSKUvg0p5fQvPbSzl+LG5LFs40e04Ip6j0pdBIRx2WLFmJ7/46zbagmFuPt+4HUnEk1T6EvfCYYcrHl3HO8WVzBiVzb9fOofCCXluxxLxJJW+xL37Xt/OO8WV/NP501l+9lR8Pp/bkUQ8S7tsSlx76cMD/PLV7YzKSef6RVNU+CJdUOlL3Npf08wNv32PUTnpPPed00lO0tNZpCv6K5G49e//FzlU7Z1LZlOQk+5yGpH4oNKXuFVS2cT8iXmcP3uU21FE4oZKX+JSU1sQe7Ceyfk6zaFIT2jvHYk7+2ua+elLW2kLhblgjrbyRXpCpS9xxXEcvvzwWkprmrly4QQW60BqIj2i0pe4squikdKaZq4+fRI/+twst+OIxB3N6UvcCIUdrvz1egC+cspYl9OIxCeVvsSNu1+27K1q5vMnjGHGqBy344jEJU3viOc5jsPPX7Y8vLqYhZOHc/eXj3c7kkjcUumL560s2sdDq4r56vzx3PWFOST5dagFkd5S6Yunvbuzkjtf2MKMUdn8+NI5OraOSB9pTl88yx6s52uPrKMgN53HrjpFhS8SAyp98ayfvhQgFHZ4ZNk8xgzNcDuOyKCg0hdPamkP8YY9xIxR2UwcoUMtiMSKSl88py0Y5rL/fheAy04Z53IakcFFb+SK5zz29i4+2FvDnUtm6+TmIjGm0hdPeXtPIz9dtZMTxuZyxYIJbscRGXQ0vSOeUXyogYfWVWAKsvnNtxbg1/74IjGnLX3xhOc/2M/3freRFL+P+z43i5z0FLcjiQxKKn1xXVNbkB//OYApyOYHZwzl9Kkj3I4kMmhpekdc979/28vBuhbu+PxsCoZoC1+kP6n0xVXNbSHuf2MHJ40fysIpw92OIzLoqfTFVY+9vYuKhjauXzTF7SgiCUGlL65pagvyX6uLWTQ9nwtm61y3IgPhmG/kGmNSgMeAiUAacBewBXgccIDNwHJrbdgYcztwCRAEbrLWrjfGTO1sbL88Eok7d76whbqWINctmux2FJGE0dWW/hVApbX2TOAi4H7gXuDW6DIfsMQYczKwCFgALAUeiN7+U2Nj/xAkHr0WKOP3f9vLN8+YxGlTtLeOyEDpqvSfAW7rcD0IFAKro9dfAs4FzgBesdY61toSINkYk3+UsZLggqEwtz//EdMLhnDLhTPcjiOSUI45vWOtbQAwxmQDK4FbgXustU50SD2QC+QAlR1u+vFyXydjjykcDhMIBHryGDrV0tISk/XEktcyuZXn3b2N7Ktu5uYz8inebj2R6Wi8lge8l8lrecB7mbyUp8sPZxljxgHPAg9aa582xvy8w7ezgRqgLnr5yOXhTpYdk9/vZ+bMmd2IfmyBQCAm64klr2VyI09zW4hrnl/NlPwsrr2wkLTkJNczHYvX8oD3MnktD3gv00DkKSoq6ta4Y07vGGMKgFeAW6y1j0UXbzTGLI5evghYA7wNXGCM8RtjxgN+a23FUcZKAnt4dTH7qpu56wtzP1X4ItL/utrS/yEwDLjNGPPx3P6NwH3GmFQgAKy01oaMMWuAtUT+kSyPjr0ZWNFxbKwfgMSP360v4VevbefEcfoglohbuprTv5FIyR9pUSdj7wDuOGLZts7GSmL69du7yElP5ulrFrgdRSRh6cNZMiB+u24P28oauObMyWSm6jh/Im5R6Uu/K69r4f89u5mhmSlcc5Y+iCXiJpW+9LsXNx0AYMWyeaSn6M1bETep9KVf1Ta188Ta3cwYlc0pE/PcjiOS8FT60m/CYYebn3mf0upm/ul843YcEUGlL/1oZdE+Xg2U84OLZ3LurAK344gIKn3pR2/YcsYOy+Dq0ye6HUVEolT60i9agyHeK6lmzphcfD6f23FEJEqlL/3iwTeKKatr5bJTxrkdRUQ6UOlLv3htaxkZKUksmp7vdhQR6UClLzEXOFDH5tI6vveZafj9mtoR8RKVvsTcw6uL8fnQ1I6IB6n0JaZ2Hmrguff38/kTxpCXlep2HBE5gkpfYurVQBmAPowl4lEqfYkZx3F4ZsM+Tho/lHF5mW7HEZFOqPQlZt4rqWZ7eQOXzdNcvohXqfQlZh59K3KSlM+dMMbtKCJyFCp9iYnKhlb+uqWMr8wbR1aaTpIi4lUqfYmJB1cV0x5yWDpfUzsiXqbSlz5rbA3y6Fu7KJwwjKkjs92OIyLHoNKXPnvu/f0ALD97istJRKQrKn3pk5qmNlas2cmkEVmcbUa6HUdEuqB33KTXGluDnHvvm1Q2tvL0t07VIZRF4oC29KXXnnp3DxUNrdz9pRNYOGW423FEpBtU+tIr+6qbeHBVMYum5/OlwrFuxxGRblLpS49VN7ax7NH1tIfC3PbZmW7HEZEeUOlLjziOwy1/2MTe6iYeWTZPu2iKxBmVvvTIn94v5ZUtZfzzBYbTpo5wO46I9JBKX3rkqXdLGDEkjW+eMdntKCLSCyp96bYP9tZQtKeaa8+aRJJOgygSl1T60m2/fHUbGSlJXDZvvNtRRKSXVPrSLW3BMBv31nDCuFxyM1PcjiMivaTSl255+aOD1DS1c+1ZmssXiWcqfemW597fz6icdBZP1/F1ROKZSl+6tH5XFa8Gyjht6nD8egNXJK5164BrxpgFwM+stYuNMVOBxwEH2Awst9aGjTG3A5cAQeAma+36o42N/cOQ/rLzUAPXPLmB/Ow0vn/udLfjiEgfdbmlb4z5F+ARID266F7gVmvtmYAPWGKMORlYBCwAlgIPHG1sbONLf/vG43+jpT3E099awLi8TLfjiEgfdWd6pxj4hw7XC4HV0csvAecCZwCvWGsda20JkGyMyT/KWIkTT7yzmz2VTdxy4QymFehwCyKDQZfTO9baPxhjJnZY5LPWOtHL9UAukANUdhjz8fLOxh5TOBwmEAh0I/qxtbS0xGQ9seS1TMfK0x5yePD1veSk+TllWPOA5Y6nn5FbvJbJa3nAe5m8lKc3J1HpOCefDdQAddHLRy7vbOwx+f1+Zs7s+5EbA4FATNYTS17LdKw8K4v2UdYQ5NEr5zF3ZoEnMrnBa3nAe5m8lge8l2kg8hQVFXVrXG/23tlojFkcvXwRsAZ4G7jAGOM3xowH/NbaiqOMFY9zHIc7X/gIU5DNOTO0i6bIYNKbLf2bgRXGmFQgAKy01oaMMWuAtUT+kSw/2tgYZJZ+9lqgnLqWIN8++zidAlFkkOlW6VtrdwOnRi9vI7KnzpFj7gDuOGJZp2PF237yUoDM1CSuOm2i21FEJMb04Sz5hP/dsJfiQ41884xJpKckuR1HRGJMpS+HOY7DA2/sYNrIIVy/aIrbcUSkH6j05bBnNuxjT2UT1y+aQlZab97uERGvU+nLYa9tLWN0bjqXnnSc21FEpJ+o9AWAyoZW3tlRyYJJeTqomsggptfwCe6vW8p46t09BA7U0dQe4jvnTHU7koj0I5V+grvnZcuhhlbmHpfLlwrHMnWkjrEjMpip9BPY5rJmbFk9/7ZkNl9fONHtOCIyADSnn8A2lDaT7PfxxcKxbkcRkQGi0k9gm8tamHNcLpmpesEnkihU+glqb1UTH5W3sGBSnttRRGQAqfQT1J0vbiHZD1ecOsHtKCIygFT6CWhvVRN/3VLGpbNydQpEkQSj0k9Az2zYC8DF03NcTiIiA02ln2B++eo27nt9B2dOG8Go7BS344jIAFPpJ5A/vrePX766nbNNPvdffrLbcUTEBSr9BBEOO9z/xg6GZqbwwNdOJjdDW/kiiUg7aCeIB97Ywc5Djfxq6YnaL18kgemvf5BrD4V5aFUx9/51G7PH5HDRnNFuRxIRF6n0B7HSmmZueKqITftqmXNcDs9cdxqpyZrRE0lkKv1Bqr6lnctXvMuh+lZ+ePEMrjxtImnJOuetSKJT6Q9CbcEwyx5bz57KJp68ej5nTc93O5KIeIRe6w8ytc3tfPOJv7GxpIY7l8xW4YvIJ2hLfxB5c9shfvDHDymtaeaq0yayTMfIF5EjqPQHgcbWIDf89j3e3HaI3IwUnrh6Pou0hS8inVDpx7lN+2r4yf9tZe3OSq5bNJnvnjONIWn6tYpI59QOcWrL/jpue24zRXuqAfjhxTO49qwpLqcSEa9T6ceRtmCYtTsreae4gl+/tZvczBS+e85UvjJvnA6RLCLdotKPA8FQmD+8t4/7XttBaU0zAOfPKuBfl8xmdG6Gy+lEJJ6o9D3uhQ/28+CqYgIH6hielcpP/2Eu58wcycjsdLejiUgcUul7UFNbkP/78CC/WbubD/bVMiwzhVsunMH1iybj8/ncjicicUyl7xGO47Bq2yEee2sXa7ZXADAqJ51/PG861y2arEMoiEhMqPRdVNPUxqZ9tTy9roTXt5bTFgqTlZrEdWdN5tTJw1ls8rVlLyIxpdIfIMFQmMa2MLsqGtmwu4rXt5bz8kcHCTuQkuTj/NmjWDw9n0uOH63j3YtIv1G7xJDjONQ2t7OxpIZDDa3sq2qiuKKR4vIGth6sj47aDUBGShKnTx3BV+ePZ+Hk4QzLSnUtt4gkjn4vfWOMH3gQOAFoBb5lrd3R3/fbV6GwQ3soTF1zOy3tYepa2mlqC9HSHuJgXQsNLUEO1DZT3xKksrGN8roWSqqaqG5q/8R6RuWkM2VkFt9ePIX2hmomjz+O8XmZnDp5OEl+Td2IyMAaiC39LwDp1tqFxphTgV8AS/rrztqCYV7+6CCrN1WStW0zjW0hapvbqWhoJRR2CDsOoXBkq/zj62EncruqxjaC4TDBsIPjdH1fqcl+hmakMDQzhYKcdC6aO5rxeZlMHJ7JzNE5ZKUlM2JI2uHxgUCAmTPH99dDFxHp0kCU/hnAXwCste8aY+b1x53Ut7TzzSc28FFpLY1tIQByM5rITE0iOz2ZoZmp5GYkkeTz4fP5SPKD3+fD7/fh9/lI8sHwIWmkJvtJ9vtI9vtJTvKRk5FCZkoSQ9KTGZKWTFqyn7ysVPKyUslOT9HWuojElYEo/RygtsP1kDEm2Vob7GxwOBwmEAj0+E5ag2EKUoNkj81gTkE6C8ekkDukN59WdaJf4ej1DtM17ZGv1mY4UAEHerjmlpaWXj22/uK1POC9TF7LA97L5LU84L1MXsozEKVfB2R3uO4/WuED+P1+Zs6c2as7+s+5f78cmUrp3Xr6i9cyeS0PeC+T1/KA9zJ5LQ94L9NA5CkqKurWuIE4c9bbwMUA0Tn9DwfgPkVEpBMDsaX/LHCeMeYdwAd8YwDuU0REOtHvpW+tDQPX9/f9iIhI13RidBGRBKLSFxFJICp9EZEEotIXEUkgKn0RkQTic7pzkJkBVFRUdAjY43YOEZE4M6GwsDC/q0GeK30REek/mt4REUkgKn0RkQSi0hcRSSAqfRGRBKLSFxFJIHF1YnRjTArwGDARSAPuArYAjxM588lmYLm1NmyMuR24BAgCN1lr1xtjTgT+EwgROV/vMmttmZuZOqzncuC71tqFbuYxxowEVgDDgCQiP6NilzOdCDwcXbaNyHmWw0feT3/kiY6fCvzJWjsnen0E8DSQAewHvmGtbeptnhhlGh+9fTKRo9lea621buXpsJ6zgN9aa8f1NkusMhljsoCHgElAKpG/t/X0Uox+Z78h8vuqAi7v6/OoO+JtS/8KoNJaeyZwEXA/cC9wa3SZD1hijDkZWAQsAJYCD0Rv/ysiv+jFwB+BWzyQiWipfTM61u08PyfyR3oWcCswwwOZbgfutNaeQeSP65KByANgjPk68HtgRIfb/wh4Ojp2I3BdH/PEItO/AfdHn9s/Bn7ich6MMeOAm4GUPmaJVaZ/BjZHx14DGJfzfB/4n+jf2kdEOqDfxVvpPwPc1uF6ECgEVkevvwScS+S8vK9Yax1rbQmQbIzJB5Zaa9+Pjk0GWtzOZIwZDvwUuCkGWfqcBzgdGGuMeRX4GrDKA5k2AnnGGB+Rs7B1OIdlv+YBqCbyj6ijw+d9PmKsm5luBv4cvRyL53af8hhj0om8Ovt2H3PELBNwAdBmjHk5up6XXc7zPpFX1BA5rWxfn9fdElelb61tsNbqPCGxAAACkklEQVTWG2OygZVEtkR91tqPP2FWD+Ty6fPy1gO51toDAMaY04DvAP/hcqY84FEi//Hr+5olBnlyibxUrbbWnguUEINXQzHItB24DwgABfTxH1EP8mCtfdFa23jEKjrmPDzWzUzW2gprbbsxxgD3AP/qZh4iW733WGtL+5IjxplGAMOstRcALxD5ObmZZx/wHWPMR0ReKTzTlzzdFVelD4dfMr4B/MZa+zR/P4M5RLYCa/j0eXk/Xo4x5jIiWyCXWGsPuZwpF5hGZJ7x98AsY8wvXcxTA1QCz0eXvQDM62ueGGT6FXCmtXYG8CTwiwHKczQdc3Y1dqAyYYw5G/gT8PW+zOf3NY8xZgxwJnC7MWYVkVdpv+9rnr5kior5c7uPee4GrrLWzgZuJPLc7ndxVfrGmALgFeAWa+1j0cUbjTGLo5cvAtYQOS/vBcYYf/TNEr+1tsIYcwWRLfzF1tqdHsi03lo7OzoPuxTYYq3t0zRPX39GwFtEz2kMfDzX2CcxyFRFpGgh8sbpMPqgB3mO5vB5n7sxdkAyRQv/V8CF1toNbuax1u631hpr7eLoc7vKWrvUzUxRMX1uxyBPNX9/xdjn53V3xdXeO8APifxgbjPGfDyXdiNwnzEmlcjL/5XW2pAxZg2wlsg/tuXGmCQiUwQlwB8jr4JZba293a1Mfbzf/spzM/CIMeYGIk/Iyz2Q6VvA740xQaCNyJtw/Z7nGLe/C3jCGHMNUMEA/oyOcftfEtkj5Ynoc9taa/vyBnNf8/SHvmb6MZHn9loi8+fLXM7zXeD+aDf56L9O+AQdcE1EJIHE1fSOiIj0jUpfRCSBqPRFRBKISl9EJIGo9EVEEohKX0Qkgaj0RUQSiEpfRCSB/H8Y7sbT0fualQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "daily_df['sha1'].cumsum().plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User contributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_user_contributions(username,lang='en',start=pd.Timestamp('2015-01-01'),stop=pd.Timestamp('2017-11-09'),skip_power=True):\n",
    "    \"\"\"Takes Wikipedia username and returns a DataFrame of revisions\n",
    "    \n",
    "    username - a string with the title of the page on Wikipedia\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition,\n",
    "        defaults to \"en\"\n",
    "    start - a datetime or Timestamp for the earliest user contributions to retrieve\n",
    "    stop - a datetime or Timestamp for the latest user contributions to retrieve\n",
    "    skip_power = If True, skips users who made more than 500 edits in a month\n",
    "        \n",
    "    Returns:\n",
    "    revision_list - a DataFrame containing the revision meta-data such as \n",
    "        parentid, revid,sha1, size, timestamp, and user name\n",
    "        \n",
    "    API endpoint docs: https://www.mediawiki.org/wiki/API:Usercontribs\n",
    "    \"\"\"\n",
    "    \n",
    "    start_utc = datetime.strftime(start, '%Y-%m-%dT%H:%M:%SZ')\n",
    "    stop_utc = datetime.strftime(stop, '%Y-%m-%dT%H:%M:%SZ')\n",
    "    \n",
    "    revision_list = list()\n",
    "    \n",
    "    # Configure the parameters\n",
    "    params = {'action':'query',\n",
    "              'list':'usercontribs',\n",
    "              'ucuser':username,\n",
    "              'ucprop':'ids|title|comment|timestamp|flags|size|sizediff',\n",
    "              'ucstart':start_utc,\n",
    "              'ucstop':stop_utc,\n",
    "              'uclimit':500,\n",
    "              'ucdir':'newer',\n",
    "              'format':'json',\n",
    "              'formatversion':2\n",
    "             }\n",
    "    \n",
    "    # Point to the API\n",
    "    url = 'https://{0}.wikipedia.org/w/api.php'.format(lang)\n",
    "    \n",
    "    # Make the request\n",
    "    json_response = requests.get(url,params=params).json()\n",
    "    \n",
    "    if 'query' in json_response:\n",
    "        subquery_revision_list = json_response['query']['usercontribs']\n",
    "        revision_list += subquery_revision_list\n",
    "    else:\n",
    "        raise KeyError(\"There is no 'query' key present in the API response.\")\n",
    "\n",
    "    # If the first 500 edits took place in less than 30 days, we've got ourselves a power user, bot, or cyborg\n",
    "    earliest_first_500 = pd.to_datetime(json_response['query']['usercontribs'][0]['timestamp'])\n",
    "    latest_first_500 = pd.to_datetime(json_response['query']['usercontribs'][-1]['timestamp'])\n",
    "    days_elapsed_first_500 = latest_first_500 - earliest_first_500\n",
    "    \n",
    "    if len(subquery_revision_list) == 500 and days_elapsed_first_500 > np.timedelta64(30,'D'):\n",
    "    \n",
    "        revision_list += subquery_revision_list\n",
    "\n",
    "        while True:\n",
    "\n",
    "            if 'continue' not in json_response:\n",
    "                break\n",
    "\n",
    "            else:\n",
    "                query_continue = json_response['continue']['uccontinue']\n",
    "                \n",
    "                continue_params = {'action':'query',\n",
    "                                   'list':'usercontribs',\n",
    "                                   'ucuser':username,\n",
    "                                   'ucprop':'ids|title|comment|timestamp|flags|size|sizediff',\n",
    "                                   'ucstart':start_utc,\n",
    "                                   'ucstop':stop_utc,\n",
    "                                   'uclimit':500,\n",
    "                                   'uccontinue':query_continue,\n",
    "                                   'ucdir':'newer',\n",
    "                                   'format':'json',\n",
    "                                   'formatversion':2\n",
    "                                  }\n",
    "                # Make the request\n",
    "                json_response = requests.get(url,params=continue_params).json()\n",
    "                \n",
    "                \n",
    "                subquery_revision_list = json_response['query']['usercontribs']\n",
    "                revision_list += subquery_revision_list\n",
    "    \n",
    "    elif 'continue' not in json_response:\n",
    "        \n",
    "        revision_list += subquery_revision_list\n",
    "        \n",
    "    df = pd.DataFrame(revision_list)\n",
    "\n",
    "    df['timestamp'] = pd.to_datetime(df['timestamp'])\n",
    "    df['date'] = df['timestamp'].apply(lambda x:x.date())\n",
    "    df['userid'] = df['userid'].fillna(0).apply(lambda x:str(int(x)))\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_user_info(username_list,lang='en'):\n",
    "    \"\"\"Takes a list of Wikipedia usernames and returns a JSON of their information\n",
    "    \n",
    "    username_list - a list of strings for all the usernames\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition,\n",
    "        defaults to \"en\"\n",
    "        \n",
    "    Returns:\n",
    "    users_info - a list of information about users\n",
    "    \n",
    "    API endpoint docs: https://www.mediawiki.org/wiki/API:Users\n",
    "    \"\"\"\n",
    "    users_info = []\n",
    "    \n",
    "    chunked_username_list = chunk_list(username_list)\n",
    "    \n",
    "    for chunk in chunked_username_list:\n",
    "        usernames = '|'.join(chunk)\n",
    "        \n",
    "        # Configure the parameters\n",
    "        params = {'action':'query',\n",
    "                  'list':'users',\n",
    "                  'ususers':usernames,\n",
    "                  'usprop':'blockinfo|groups|editcount|registration|gender',\n",
    "                  'format':'json',\n",
    "                  'formatversion':2\n",
    "                 }\n",
    "    \n",
    "        # Point to the API\n",
    "        url = 'https://{0}.wikipedia.org/w/api.php'.format(lang)\n",
    "    \n",
    "        # Make the request\n",
    "        json_response = requests.get(url,params=params).json()\n",
    "        \n",
    "        'usprop=&format=json&formatversion=2\".format(usernames,lang)\n",
    "        json_response = requests.get(query_string).json()\n",
    "        if 'query' in json_response:\n",
    "            users_info += json_response['query']['users']\n",
    "    \n",
    "    return users_info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inter-language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_interlanguage_links(page_title,lang='en'):\n",
    "    \"\"\"The function accepts a page_title and returns a dictionary containing \n",
    "    the title of the page in its other languages\n",
    "       \n",
    "    page_title - a string with the title of the page on Wikipedia\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition, \n",
    "        defaults to \"en\"\n",
    "       \n",
    "    Returns:\n",
    "    langlink_dict - a dictionary keyed by lang codes and page title as values\n",
    "    \"\"\"\n",
    "    \n",
    "    params = {'action':'query',\n",
    "              'prop':'langlinks',\n",
    "              'titles':page_title,\n",
    "              'llprop':'autonym|langname',\n",
    "              'lllimit':500,\n",
    "              'format':'json'\n",
    "             }\n",
    "    \n",
    "    # Point to the API\n",
    "    url = 'https://{0}.wikipedia.org/w/api.php'.format(lang)\n",
    "\n",
    "    # Make the request\n",
    "    json_response = requests.get(url,params=params).json()\n",
    "    \n",
    "    interlanguage_link_dict = dict()\n",
    "    interlanguage_link_dict['en'] = page_title\n",
    "\n",
    "    if 'langlinks' in json_response['query']['pages'][0]:\n",
    "        langlink_dict = json_response['query']['pages'][0]['langlinks']\n",
    "\n",
    "        for d in langlink_dict:\n",
    "            lang = d['lang']\n",
    "            title = d['title']\n",
    "            interlanguage_link_dict[lang] = title\n",
    "            \n",
    "    return interlanguage_link_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_interlanguage_revisions(page_title,lang='en'):\n",
    "    \"\"\"Takes a Wikipedia page title and return the interlanguage revision history\n",
    "    \n",
    "    page_title - a string with the title of the page on Wikipedia\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition,\n",
    "        defaults to \"en\"\n",
    "    \n",
    "    Returns:\n",
    "    extlinks_per_lang - a dictionary keyed by language returning a dictionary\n",
    "        keyed by page title returning a Counter dictionary of external links'\n",
    "        top-level domains and counts\n",
    "    \"\"\"\n",
    "    revisions_df_dict = {}\n",
    "\n",
    "    language_titles = get_interlanguage_links(page_title,lang)\n",
    "\n",
    "    for lang,title in language_titles.items():\n",
    "        try:\n",
    "            revisions_df_dict[lang] = get_page_revisions(title,lang)\n",
    "        \n",
    "        except KeyboardInterrupt:\n",
    "            raise\n",
    "            \n",
    "        except:\n",
    "            print(\"Error getting revisions in {0} version of \\\"{1}\\\"\".format(lang,title))\n",
    "            pass\n",
    "    \n",
    "    concat_df = pd.concat(revisions_df_dict.values(),keys=revisions_df_dict.keys(),\n",
    "                          names=['lang','rev_num']).reset_index()\n",
    "    \n",
    "    return concat_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### External links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rev_externallinks(revid,lang='en',redirects=1):\n",
    "    \"\"\"Takes a revision id and returns a list of external links on the revision\n",
    "    \n",
    "    revid - a numeric revision id as a string\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language \n",
    "        edition, defaults to \"en\"\n",
    "    redirects - 1 or 0 for whether to follow page redirects, defaults to 1\n",
    "    parse - 1 or 0 for whether to return the raw HTML or paragraph text\n",
    "    \n",
    "    Returns:\n",
    "    str - a list of strings with the URLs\n",
    "    \"\"\"\n",
    "    \n",
    "    # Get the response from the API for a query\n",
    "    \n",
    "    params = {'action':'parse',\n",
    "              'prop':'langlinks',\n",
    "              'oldid':revid,\n",
    "              'redirects':redirects,\n",
    "              'prop':'externallinks',\n",
    "              'disableeditsection':1,\n",
    "              'disabletoc':1,\n",
    "              'format':'json',\n",
    "             }\n",
    "    \n",
    "    # Point to the API\n",
    "    url = 'https://{0}.wikipedia.org/w/api.php'.format(lang)\n",
    "\n",
    "    # Make the request\n",
    "    json_response = requests.get(url,params=params).json()\n",
    "    \n",
    "    if 'parse' in json_response.keys():\n",
    "        if 'externallinks' in json_response['parse']:\n",
    "            return json_response['parse']['externallinks']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_rev_content(revid,lang='en',redirects=1,parsed_text=1):\n",
    "    \"\"\"Takes a revision id and returns a (large) string of the HTML content \n",
    "    of the revision.\n",
    "    \n",
    "    revid - a numeric revision id as a string\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language \n",
    "        edition, defaults to \"en\"\n",
    "    redirects - 1 or 0 for whether to follow page redirects, defaults to 1\n",
    "    parse - 1 or 0 for whether to return the raw HTML or paragraph text\n",
    "    \n",
    "    Returns:\n",
    "    str - a (large) string of the content of the revision\n",
    "    \"\"\"\n",
    "    \n",
    "    bad_titles = ['Special:','Wikipedia:','Help:','Template:','Category:','International Standard','Portal:','s:','File:','Digital object identifier','(page does not exist)']\n",
    "    \n",
    "    # Get the response from the API for a query\n",
    "    # After passing a page title, the API returns the HTML markup of the current article version within a JSON payload\n",
    "    req = requests.get('https://{2}.wikipedia.org/w/api.php?action=parse&format=json&oldid={0}&redirects={1}&prop=text&disableeditsection=1&disabletoc=1'.format(revid,redirects,lang))\n",
    "    \n",
    "    # Read the response into JSON to parse and extract the HTML\n",
    "    json_string = json.loads(req.text)\n",
    "    \n",
    "    if 'parse' in json_string.keys():\n",
    "        page_html = json_string['parse']['text']['*']\n",
    "\n",
    "        # Parse the HTML into Beautiful Soup\n",
    "        soup = BeautifulSoup(page_html,'lxml')\n",
    "        \n",
    "        # Remove sections at end\n",
    "        bad_sections = ['See_also','Notes','References','Bibliography','External_links']\n",
    "        sections = soup.find_all('h2')\n",
    "        for section in sections:\n",
    "            if section.span['id'] in bad_sections:\n",
    "                \n",
    "                # Clean out the divs\n",
    "                div_siblings = section.find_next_siblings('div')\n",
    "                for sibling in div_siblings:\n",
    "                    sibling.clear()\n",
    "                    \n",
    "                # Clean out the ULs\n",
    "                ul_siblings = section.find_next_siblings('ul')\n",
    "                for sibling in ul_siblings:\n",
    "                    sibling.clear()\n",
    "        \n",
    "        # Get all the paragraphs\n",
    "        paras = soup.find_all('p')\n",
    "        \n",
    "        text_list = []\n",
    "        \n",
    "        for para in paras:\n",
    "            if parsed_text:\n",
    "                _s = para.text\n",
    "                # Remove the citations\n",
    "                _s = re.sub(r'\\[[0-9]+\\]','',_s)\n",
    "                text_list.append(_s)\n",
    "            else:\n",
    "                text_list.append(str(para))\n",
    "        \n",
    "        return '\\n'.join(text_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_rev_markup(revid,lang='en',redirects=1,parsed_text=1):\n",
    "    \"\"\"Takes a revision id and returns a (large) string of the HTML content \n",
    "    of the revision.\n",
    "    \n",
    "    revid - a numeric revision id as a string\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language \n",
    "        edition, defaults to \"en\"\n",
    "    redirects - 1 or 0 for whether to follow page redirects, defaults to 1\n",
    "    parse - 1 or 0 for whether to return the raw HTML or paragraph text\n",
    "    \n",
    "    Returns:\n",
    "    str - a (large) string of the content of the revision\n",
    "    \"\"\"\n",
    "    \n",
    "    bad_titles = ['Special:','Wikipedia:','Help:','Template:','Category:','International Standard','Portal:','s:','File:','Digital object identifier','(page does not exist)']\n",
    "    \n",
    "    # Get the response from the API for a query\n",
    "    # After passing a page title, the API returns the HTML markup of the current article version within a JSON payload\n",
    "    req = requests.get('https://{2}.wikipedia.org/w/api.php?action=parse&format=json&oldid={0}&redirects={1}&prop=text&disableeditsection=1&disabletoc=1'.format(revid,redirects,lang))\n",
    "    \n",
    "    # Read the response into JSON to parse and extract the HTML\n",
    "    json_string = json.loads(req.text)\n",
    "    \n",
    "    if 'parse' in json_string.keys():\n",
    "        page_html = json_string['parse']['text']['*']\n",
    "\n",
    "        # Parse the HTML into Beautiful Soup\n",
    "        soup = BeautifulSoup(page_html,'lxml')\n",
    "        \n",
    "        return str(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_rev_outlinks(revid,lang='en',redirects=1):\n",
    "    \"\"\"Takes a page title and returns a list of wiki-links on the page. The \n",
    "    list may contain duplicates and the position in the list is approximately \n",
    "    where the links occurred.\n",
    "    \n",
    "    revid - a numeric revision id as a string\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language \n",
    "        edition, defaults to \"en\"\n",
    "    redirects - 1 or 0 for whether to follow page redirects, defaults to 1\n",
    "    \n",
    "    Returns:\n",
    "    outlinks_per_lang - a dictionary keyed by language returning a dictionary \n",
    "        keyed by page title returning a list of outlinks\n",
    "    \"\"\"\n",
    "        \n",
    "    bad_titles = ['Special:','Wikipedia:','Help:','Template:','Category:','International Standard','Portal:','s:','File:','Digital object identifier','(page does not exist)']\n",
    "    \n",
    "    # Get the response from the API for a query\n",
    "    # After passing a page title, the API returns the HTML markup of the current article version within a JSON payload\n",
    "    req = requests.get('https://{2}.wikipedia.org/w/api.php?action=parse&format=json&oldid={0}&redirects={1}&prop=text&disableeditsection=1&disabletoc=1'.format(revid,redirects,lang))\n",
    "    \n",
    "    # Read the response into JSON to parse and extract the HTML\n",
    "    json_string = json.loads(req.text)\n",
    "    \n",
    "    # Initialize an empty list to store the links\n",
    "    outlinks_list = [] \n",
    "    \n",
    "    if 'parse' in json_string.keys():\n",
    "        page_html = json_string['parse']['text']['*']\n",
    "\n",
    "        # Parse the HTML into Beautiful Soup\n",
    "        soup = BeautifulSoup(page_html,'lxml')\n",
    "        \n",
    "        # Remove sections at end\n",
    "        bad_sections = ['See_also','Notes','References','Bibliography','External_links']\n",
    "        sections = soup.find_all('h2')\n",
    "        for section in sections:\n",
    "            if section.span['id'] in bad_sections:\n",
    "                \n",
    "                # Clean out the divs\n",
    "                div_siblings = section.find_next_siblings('div')\n",
    "                for sibling in div_siblings:\n",
    "                    sibling.clear()\n",
    "                    \n",
    "                # Clean out the ULs\n",
    "                ul_siblings = section.find_next_siblings('ul')\n",
    "                for sibling in ul_siblings:\n",
    "                    sibling.clear()\n",
    "\n",
    "        # Delete tags associated with templates\n",
    "        for tag in soup.find_all('tr'):\n",
    "            tag.replace_with('')\n",
    "\n",
    "        # For each paragraph tag, extract the titles within the links\n",
    "        for para in soup.find_all('p'):\n",
    "            for link in para.find_all('a'):\n",
    "                if link.has_attr('title'):\n",
    "                    title = link['title']\n",
    "                    # Ignore links that aren't interesting or are redlinks\n",
    "                    if all(bad not in title for bad in bad_titles) and 'redlink' not in link['href']:\n",
    "                        outlinks_list.append(title)\n",
    "\n",
    "        # For each unordered list, extract the titles within the child links\n",
    "        for unordered_list in soup.find_all('ul'):\n",
    "            for item in unordered_list.find_all('li'):\n",
    "                for link in item.find_all('a'):\n",
    "                    if link.has_attr('title'):\n",
    "                        title = link['title']\n",
    "                        # Ignore links that aren't interesting or are redlinks\n",
    "                        if all(bad not in title for bad in bad_titles) and 'redlink' not in link['href']:\n",
    "                            outlinks_list.append(title)\n",
    "\n",
    "    return outlinks_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_page_outlinks(page_title,lang='en',redirects=1):\n",
    "    \"\"\"Takes a page title and returns a list of wiki-links on the page. The \n",
    "    list may contain duplicates and the position in the list is approximately \n",
    "    where the links occurred.\n",
    "    \n",
    "    page_title - a string with the title of the page on Wikipedia\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language \n",
    "        edition, defaults to \"en\"\n",
    "    redirects - 1 or 0 for whether to follow page redirects, defaults to 1\n",
    "    \n",
    "    Returns:\n",
    "    outlinks_per_lang - a dictionary keyed by language returning a dictionary \n",
    "        keyed by page title returning a list of outlinks\n",
    "    \"\"\"\n",
    "    \n",
    "    # Replace spaces with underscores\n",
    "    page_title = page_title.replace(' ','_')\n",
    "    \n",
    "    bad_titles = ['Special:','Wikipedia:','Help:','Template:','Category:','International Standard','Portal:','s:','File:','Digital object identifier','(page does not exist)']\n",
    "    \n",
    "    # Get the response from the API for a query\n",
    "    # After passing a page title, the API returns the HTML markup of the current article version within a JSON payload\n",
    "    req = requests.get('https://{2}.wikipedia.org/w/api.php?action=parse&format=json&page={0}&redirects={1}&prop=text&disableeditsection=1&disabletoc=1'.format(page_title,redirects,lang))\n",
    "    \n",
    "    # Read the response into JSON to parse and extract the HTML\n",
    "    json_string = json.loads(req.text)\n",
    "    \n",
    "    # Initialize an empty list to store the links\n",
    "    outlinks_list = [] \n",
    "    \n",
    "    if 'parse' in json_string.keys():\n",
    "        page_html = json_string['parse']['text']['*']\n",
    "\n",
    "        # Parse the HTML into Beautiful Soup\n",
    "        soup = BeautifulSoup(page_html,'lxml')\n",
    "        \n",
    "        # Remove sections at end\n",
    "        bad_sections = ['See_also','Notes','References','Bibliography','External_links']\n",
    "        sections = soup.find_all('h2')\n",
    "        for section in sections:\n",
    "            if section.span['id'] in bad_sections:\n",
    "                \n",
    "                # Clean out the divs\n",
    "                div_siblings = section.find_next_siblings('div')\n",
    "                for sibling in div_siblings:\n",
    "                    sibling.clear()\n",
    "                    \n",
    "                # Clean out the ULs\n",
    "                ul_siblings = section.find_next_siblings('ul')\n",
    "                for sibling in ul_siblings:\n",
    "                    sibling.clear()\n",
    "        \n",
    "        # Delete tags associated with templates\n",
    "        for tag in soup.find_all('tr'):\n",
    "            tag.replace_with('')\n",
    "\n",
    "        # For each paragraph tag, extract the titles within the links\n",
    "        for para in soup.find_all('p'):\n",
    "            for link in para.find_all('a'):\n",
    "                if link.has_attr('title'):\n",
    "                    title = link['title']\n",
    "                    # Ignore links that aren't interesting or are redlinks\n",
    "                    if all(bad not in title for bad in bad_titles) and 'redlink' not in link['href']:\n",
    "                        outlinks_list.append(title)\n",
    "\n",
    "        # For each unordered list, extract the titles within the child links\n",
    "        for unordered_list in soup.find_all('ul'):\n",
    "            for item in unordered_list.find_all('li'):\n",
    "                for link in item.find_all('a'):\n",
    "                    if link.has_attr('title'):\n",
    "                        title = link['title']\n",
    "                        # Ignore links that aren't interesting or are redlinks\n",
    "                        if all(bad not in title for bad in bad_titles) and 'redlink' not in link['href']:\n",
    "                            outlinks_list.append(title)\n",
    "\n",
    "    return outlinks_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_category_memberships(page_title,lang='en'):\n",
    "    \"\"\"The function accepts a page_title and returns a list of categories\n",
    "    the page is a member of\n",
    "    \n",
    "    category_title - a string of the page name\n",
    "    \n",
    "    Returns:\n",
    "    members - a list containing strings of the categories of which the page is a mamber\n",
    "    \n",
    "    \"\"\"\n",
    "    _S=\"https://{1}.wikipedia.org/w/api.php?action=query&prop=categories&titles={0}&clprop=timestamp&clshow=!hidden&cllimit=500&format=json&formatversion=2\".format(page_title,lang)\n",
    "    json_response = requests.get(_S).json()\n",
    "\n",
    "    categories = list()\n",
    "\n",
    "    if 'pages' in json_response['query']:\n",
    "        if 'categories' in json_response['query']['pages'][0]:\n",
    "            for category in json_response['query']['pages'][0]['categories']:\n",
    "                title = category['title']#.split(':')[1]\n",
    "                categories.append(title)\n",
    "                #timestamp = category['timestamp']\n",
    "                #categories.append({title:timestamp})\n",
    "            \n",
    "    return categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_category_subcategories(category_title,lang='en'):\n",
    "    \"\"\"The function accepts a category_title and returns a list of the category's sub-categories\n",
    "    \n",
    "    category_title - a string (including \"Category:\" prefix) of the category name\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition,\n",
    "        defaults to \"en\"\n",
    "    \n",
    "    Returns:\n",
    "    members - a list containing strings of the sub-categories in the category\n",
    "    \n",
    "    \"\"\"\n",
    "    # Replace spaces with underscores\n",
    "    category_title = category_title.replace(' ','_')\n",
    "    \n",
    "    # Make sure \"Category:\" appears in the title\n",
    "    if 'Category:' not in category_title:\n",
    "        category_title = 'Category:' + category_title\n",
    "        \n",
    "    _S=\"https://{1}.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle={0}&cmtype=subcat&cmprop=title&cmlimit=500&format=json&formatversion=2\".format(category_title,lang)\n",
    "    json_response = requests.get(_S).json()\n",
    "\n",
    "    members = list()\n",
    "    \n",
    "    if 'categorymembers' in json_response['query']:\n",
    "        for member in json_response['query']['categorymembers']:\n",
    "            members.append(member['title'])\n",
    "            \n",
    "    return members"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_category_members(category_title,depth=1,lang='en'):\n",
    "    \"\"\"The function accepts a category_title and returns a list of category members\n",
    "    \n",
    "    category_title - a string (including \"Category:\" prefix) of the category name\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition,\n",
    "        defaults to \"en\"\n",
    "    \n",
    "    Returns:\n",
    "    members - a list containing strings of the page titles in the category\n",
    "    \n",
    "    \"\"\"\n",
    "    # Replace spaces with underscores\n",
    "    category_title = category_title.replace(' ','_')\n",
    "    \n",
    "    # Make sure \"Category:\" appears in the title\n",
    "    if 'Category:' not in category_title:\n",
    "        category_title = 'Category:' + category_title\n",
    "    \n",
    "    _S=\"https://{1}.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle={0}&cmprop=title&cmnamespace=0&cmlimit=500&format=json&formatversion=2\".format(category_title,lang)\n",
    "    json_response = requests.get(_S).json()\n",
    "\n",
    "    members = list()\n",
    "    \n",
    "    if depth < 0:\n",
    "        return members\n",
    "    \n",
    "    if 'categorymembers' in json_response['query']:\n",
    "        for member in json_response['query']['categorymembers']:\n",
    "            members.append(member['title'])\n",
    "            \n",
    "    subcats = get_category_subcategories(category_title,lang=lang)\n",
    "    \n",
    "    for subcat in subcats:\n",
    "        members += get_category_members(subcat,depth-1)\n",
    "            \n",
    "    return members"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### External links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_external_links(page_title,lang='en'):\n",
    "    external_links = list()\n",
    "    \n",
    "    query_string = \"https://{1}.wikipedia.org/w/api.php?action=query&titles={0}&prop=extlinks&ellimit=500&format=json&formatversion=2\".format(page_title,lang)\n",
    "    json_response = requests.get(query_string).json()\n",
    "    \n",
    "    if 'missing' not in json_response['query']['pages'][0] and 'extlinks' in json_response['query']['pages'][0]:\n",
    "        extlinks = json_response['query']['pages'][0]['extlinks']\n",
    "        \n",
    "        # Clean the extlinks\n",
    "        cleaned_extlinks = list()\n",
    "        \n",
    "        for l in extlinks:\n",
    "            if 'web.archive.org' in l['url']: # Internet Archives have two https in them, get the second\n",
    "                raw_url = 'http://' + l['url'].split('/http://')[1]\n",
    "            else:\n",
    "                raw_url = l['url']\n",
    "            \n",
    "            # Try to use the tldextract function, otherwise fall back to urlparse\n",
    "            try:\n",
    "                netloc = \"{0}.{1}\".format(tldextract.extract(raw_url).domain, tldextract.extract(raw_url).suffix)\n",
    "            except:\n",
    "                netloc = urlparse(raw_url).netloc\n",
    "                \n",
    "            external_links.append(netloc)\n",
    "    \n",
    "    return external_links"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Redirects linking to a page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_redirects_linking_here(page_title,lang='en',namespace=0):\n",
    "    \"\"\"Takes a page title and returns a list of redirects linking to the page\n",
    "    \n",
    "    page_title - a string with the title of the page on Wikipedia\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language \n",
    "        edition, defaults to \"en\"\n",
    "    namespace - limit to pages from a specific namespace, defaults to 0\n",
    "    \n",
    "    Returns:\n",
    "    linkshere - a list of strings with the redirect titles\n",
    "    \"\"\"\n",
    "    \n",
    "    # Get the response from the API for a query\n",
    "    # After passing a page title, the API returns the HTML markup of the current article version within a JSON payload\n",
    "    \n",
    "    lh_list = list()\n",
    "    \n",
    "    query_string = 'https://{1}.wikipedia.org/w/api.php?action=query&titles={0}&prop=linkshere&lhprop=title|redirect&lhnamespace={2}&lhshow=redirect&lhlimit=500&format=json&formatversion=2'.format(page_title,lang,namespace)\n",
    "    json_response = requests.get(query_string).json()\n",
    "    \n",
    "    if 'linkshere' in json_response['query']['pages'][0]:\n",
    "        subquery_lh_list = json_response['query']['pages'][0]['linkshere']\n",
    "        lh_list += subquery_lh_list\n",
    "    \n",
    "        while True:\n",
    "\n",
    "            if 'continue' not in json_response:\n",
    "                break\n",
    "\n",
    "            else:\n",
    "                query_continue = json_response['continue']['lhcontinue']\n",
    "                query_string = 'https://{1}.wikipedia.org/w/api.php?action=query&titles={0}&lhcontinue={3}&prop=linkshere&lhprop=title|redirect&lhnamespace={2}&lhshow=redirect&lhlimit=500&format=json&formatversion=2'.format(page_title,lang,namespace,query_continue)\n",
    "                json_response = requests.get(query_string).json()\n",
    "                subquery_lh_list = json_response['query']['pages'][0]['linkshere']\n",
    "                lh_list += subquery_lh_list\n",
    "    \n",
    "    return [i['title'] for i in lh_list]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log events "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_log_events(page_title,lang='en'):\n",
    "    \"\"\"Takes Wikipedia page title and returns a list of revisions\n",
    "    \n",
    "    page_title - a string with the title of the page on Wikipedia\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition,\n",
    "        defaults to \"en\"\n",
    "        \n",
    "    Returns:\n",
    "    revision_list - a list of dictionaries, where each dictionary is the revision\n",
    "        meta-data susch as parentid, revid,sha1, size, timestamp, and user name\n",
    "    \"\"\"\n",
    "    \n",
    "    event_list = list()\n",
    "    \n",
    "    query_string = \"https://{1}.wikipedia.org/w/api.php?action=query&letitle={0}&list=logevents&leprop=ids|title|type|user|userid|timestamp|comment|tags&lelimit=500&format=json&formatversion=2\".format(page_title,lang)\n",
    "    json_response = requests.get(query_string).json()\n",
    "\n",
    "    subquery_revision_list = json_response['query']['logevents']\n",
    "    event_list += subquery_revision_list\n",
    "    \n",
    "    while True:\n",
    "    \n",
    "        if 'continue' not in json_response:\n",
    "            break\n",
    "            \n",
    "        else:\n",
    "            query_continue = json_response['continue']['lecontinue']\n",
    "            query_string = \"https://{1}.wikipedia.org/w/api.php?action=query&letitle={0}&list=logevents&leprop=ids|title|type|user|userid|timestamp|comment|tags&lelimit=500&lecontinue={2}&format=json&formatversion=2\".format(page_title,lang,query_continue)\n",
    "            json_response = requests.get(query_string).json()\n",
    "            subquery_revision_list = json_response['query']['logevents']\n",
    "            event_list += subquery_revision_list\n",
    "    \n",
    "    df = pd.DataFrame(event_list)\n",
    "    df['page'] = page_title\n",
    "    \n",
    "    if 'timestamp' in df.columns:\n",
    "        df['timestamp'] = pd.to_datetime(df['timestamp'])\n",
    "        df['date'] = df['timestamp'].apply(lambda x:x.date())\n",
    "    \n",
    "    if 'userid' in df.columns:\n",
    "        df['userid'] = df['userid'].fillna(0).apply(lambda x:str(int(x)))\n",
    "    #df['lang'] = lang\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pageviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_pageviews(page_title,lang='en',date_from='20150701',date_to=str(datetime.today().date()).replace('-','')):\n",
    "    \"\"\"Takes Wikipedia page title and returns a all the various pageview records\n",
    "    \n",
    "    page_title - a string with the title of the page on Wikipedia\n",
    "    lang - a string (typically two letter ISO 639-1 code) for the language edition,\n",
    "        defaults to \"en\"\n",
    "        datefrom - a date string in a YYYYMMDD format, defaults to 20150701\n",
    "        dateto - a date string in a YYYYMMDD format, defaults to today\n",
    "        \n",
    "    Returns:\n",
    "    revision_list - a DataFrame indexed by date and multi-columned by agent and access type\n",
    "    \"\"\"\n",
    "    quoted_page_title = quote(page_title, safe='')\n",
    "    \n",
    "    df_list = []\n",
    "    for access in ['all-access','desktop','mobile-app','mobile-web']:\n",
    "        for agent in ['all-agents','user','spider','bot']:\n",
    "            s = \"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/{1}.wikipedia.org/{2}/{3}/{0}/daily/{4}/{5}\".format(quoted_page_title,lang,access,agent,date_from,date_to)\n",
    "            json_response = requests.get(s).json()\n",
    "            df = pd.DataFrame(json_response['items'])\n",
    "            df_list.append(df)\n",
    "\n",
    "    concat_df = pd.concat(df_list)\n",
    "    concat_df['timestamp'] = pd.to_datetime(concat_df['timestamp'],format='%Y%m%d%H')\n",
    "    concat_df = concat_df.set_index(['timestamp','agent','access'])['views'].unstack([1,2]).sort_index(axis=1)\n",
    "    concat_df[('page','page')] = page_title\n",
    "    return concat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>agent</th>\n",
       "      <th colspan=\"4\" halign=\"left\">all-agents</th>\n",
       "      <th colspan=\"4\" halign=\"left\">bot</th>\n",
       "      <th colspan=\"4\" halign=\"left\">spider</th>\n",
       "      <th colspan=\"4\" halign=\"left\">user</th>\n",
       "      <th>page</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>access</th>\n",
       "      <th>all-access</th>\n",
       "      <th>desktop</th>\n",
       "      <th>mobile-app</th>\n",
       "      <th>mobile-web</th>\n",
       "      <th>all-access</th>\n",
       "      <th>desktop</th>\n",
       "      <th>mobile-app</th>\n",
       "      <th>mobile-web</th>\n",
       "      <th>all-access</th>\n",
       "      <th>desktop</th>\n",
       "      <th>mobile-app</th>\n",
       "      <th>mobile-web</th>\n",
       "      <th>all-access</th>\n",
       "      <th>desktop</th>\n",
       "      <th>mobile-app</th>\n",
       "      <th>mobile-web</th>\n",
       "      <th>page</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-07-01</th>\n",
       "      <td>82159</td>\n",
       "      <td>35886</td>\n",
       "      <td>1292</td>\n",
       "      <td>44981</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>410</td>\n",
       "      <td>390</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>81749</td>\n",
       "      <td>35496</td>\n",
       "      <td>1292</td>\n",
       "      <td>44961</td>\n",
       "      <td>Donald Trump</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-02</th>\n",
       "      <td>83458</td>\n",
       "      <td>37554</td>\n",
       "      <td>1453</td>\n",
       "      <td>44451</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>403</td>\n",
       "      <td>365</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>83055</td>\n",
       "      <td>37189</td>\n",
       "      <td>1453</td>\n",
       "      <td>44413</td>\n",
       "      <td>Donald Trump</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-03</th>\n",
       "      <td>55615</td>\n",
       "      <td>22154</td>\n",
       "      <td>997</td>\n",
       "      <td>32464</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>441</td>\n",
       "      <td>419</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>55174</td>\n",
       "      <td>21735</td>\n",
       "      <td>997</td>\n",
       "      <td>32442</td>\n",
       "      <td>Donald Trump</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-04</th>\n",
       "      <td>43865</td>\n",
       "      <td>16640</td>\n",
       "      <td>795</td>\n",
       "      <td>26430</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>492</td>\n",
       "      <td>469</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>43373</td>\n",
       "      <td>16171</td>\n",
       "      <td>795</td>\n",
       "      <td>26407</td>\n",
       "      <td>Donald Trump</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-05</th>\n",
       "      <td>42220</td>\n",
       "      <td>16158</td>\n",
       "      <td>819</td>\n",
       "      <td>25243</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>306</td>\n",
       "      <td>291</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>41914</td>\n",
       "      <td>15867</td>\n",
       "      <td>819</td>\n",
       "      <td>25228</td>\n",
       "      <td>Donald Trump</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "agent      all-agents                                      bot          \\\n",
       "access     all-access desktop mobile-app mobile-web all-access desktop   \n",
       "timestamp                                                                \n",
       "2015-07-01      82159   35886       1292      44981          0       0   \n",
       "2015-07-02      83458   37554       1453      44451          0       0   \n",
       "2015-07-03      55615   22154        997      32464          0       0   \n",
       "2015-07-04      43865   16640        795      26430          0       0   \n",
       "2015-07-05      42220   16158        819      25243          0       0   \n",
       "\n",
       "agent                                spider                                \\\n",
       "access     mobile-app mobile-web all-access desktop mobile-app mobile-web   \n",
       "timestamp                                                                   \n",
       "2015-07-01          0          0        410     390          0         20   \n",
       "2015-07-02          0          0        403     365          0         38   \n",
       "2015-07-03          0          0        441     419          0         22   \n",
       "2015-07-04          0          0        492     469          0         23   \n",
       "2015-07-05          0          0        306     291          0         15   \n",
       "\n",
       "agent            user                                        page  \n",
       "access     all-access desktop mobile-app mobile-web          page  \n",
       "timestamp                                                          \n",
       "2015-07-01      81749   35496       1292      44961  Donald Trump  \n",
       "2015-07-02      83055   37189       1453      44413  Donald Trump  \n",
       "2015-07-03      55174   21735        997      32442  Donald Trump  \n",
       "2015-07-04      43373   16171        795      26407  Donald Trump  \n",
       "2015-07-05      41914   15867        819      25228  Donald Trump  "
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_pv = get_pageviews('Donald Trump')\n",
    "_pv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}