2020-04-21-gdpr.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GDPR Cases - python analysis\n",
    "\n",
    "| [screencast](https://youtu.be/z6xNKZZMWgU) | [notebook](https://github.com/machow/tidytuesday-py/blob/master/2020-04-21-gdpr.ipynb) |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-d279e0f00384>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mplotnine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m gdpr_violations = pd.read_csv(\n\u001b[0m\u001b[1;32m      9\u001b[0m         \u001b[0;34m'https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-04-21/gdpr_violations.tsv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0msep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\\t\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparse_dates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"date\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from siuba import _, mutate, count, filter, group_by, ungroup, head\n",
    "from siuba import add_count\n",
    "from siuba.dply.forcats import fct_reorder, fct_lump\n",
    "from siuba.experimental.datetime import floor_date\n",
    "from plotnine import *\n",
    "\n",
    "gdpr_violations = pd.read_csv(\n",
    "        'https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-04-21/gdpr_violations.tsv',\n",
    "        sep = \"\\t\", parse_dates = [\"date\"]\n",
    "    )\n",
    "\n",
    "gdpr_text = pd.read_csv(\n",
    "    'https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-04-21/gdpr_text.tsv',\n",
    "    sep = \"\\t\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Countries with most fines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    gdpr_violations >> count(_.name, sort = True)\n",
    "    >> mutate(name = fct_reorder(_.name, -_.n))\n",
    "    >> ggplot(aes(\"name\", \"n\", group = 1)) + geom_point() + geom_line()\n",
    "    + theme(axis_text_x = element_text(angle = 45, hjust = 1))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdpr_violations >> count(_.name, _.controller, sort = True)\n",
    "\n",
    "gdpr_violations >> filter(_.controller.str.contains(\"[Gg]oogle\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GDPR cases over time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    gdpr_violations\n",
    "    >> mutate(week = floor_date(_.date, \"W\"))\n",
    "    >> filter(_.week != \"1970-01-01\")\n",
    "    >> count(_.week)\n",
    "    >> mutate(ttl = _.n.cumsum())\n",
    "    >> ggplot(aes(\"week\", \"ttl\", group = 1)) + geom_line()\n",
    "    + expand_limits(y = 0)\n",
    "    + theme(axis_text_x = element_text(angle = 45, hjust = 1))\n",
    "    + labs(title = \"GDPR cases over time\", y = \"cumulative cases\")\n",
    "#    + scale_y_log10()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Size of fines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from siuba import arrange\n",
    "\n",
    "(\n",
    "    gdpr_violations\n",
    "    >> filter(_.date != \"1970-01-01\")\n",
    "    >> ggplot(aes(\"date\", \"price\"))\n",
    "    + geom_point()\n",
    "    + theme(axis_text_x=element_text(angle=45, hjust=1))\n",
    "    + labs(title = \"GDPR fine size over time\")\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Type x Price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# multiple articles violated\n",
    "# * what is type?\n",
    "# connecting article to other data\n",
    "\n",
    "#gdpr_violations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type_counts = gdpr_violations >> count(_.type, sort = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from siuba.dply.forcats import fct_lump\n",
    "\n",
    "(\n",
    "    gdpr_violations\n",
    "    >> add_count(_.type)\n",
    "    >> mutate(type = fct_lump(_.type, n = 6))\n",
    "    >> ggplot(aes(\"type\", \"price\", group=\"type\"))\n",
    "    + geom_boxplot()\n",
    "    + scale_y_log10(labels = lambda arr: [\"{:,}\".format(x) for x in arr])\n",
    "    + theme(axis_text_x=element_text(angle=45, hjust=1))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Article violated x price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from siuba import unnest, rename\n",
    "\n",
    "gdpr_articles_nested = gdpr_violations >> mutate(\n",
    "    articles=_.article_violated.str.split(\"|\"),\n",
    "    n_violated=_.articles.apply(len)\n",
    ")\n",
    "\n",
    "# example ---\n",
    "# unnest(example.loc[[1], :], \"articles\")\n",
    "gdpr_articles = (\n",
    "    gdpr_articles_nested\n",
    "    >> unnest(\"articles\")\n",
    "    >> mutate(article_number = _.articles.str.extract(r\"Art. *([0-9]+)\").astype(float))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    gdpr_articles_nested\n",
    "    >> ggplot(aes(\"n_violated\", \"price\", group=\"n_violated\"))\n",
    "    + geom_violin()\n",
    "    + scale_y_log10(labels=lambda arr: [\"{:,}\".format(x) for x in arr])\n",
    "    + geom_point(position = position_jitter(height = 0, width = .25))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdpr_articles >> count(_.article_number, sort = True) >> head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from siuba import inner_join, distinct\n",
    "\n",
    "joined_articles = (\n",
    "    gdpr_text\n",
    "    >> distinct(_.article, _.article_title)\n",
    "    >> inner_join(_, gdpr_articles, {\"article\": \"article_number\"})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sanity check that number of rows is close to gdpr_articles\n",
    "#joined_articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    joined_articles\n",
    "    >> mutate(article_title=fct_lump(_.article_title, n=6))\n",
    "    >> ggplot(aes(\"article_title\", \"price\", \"article_title\")) + geom_boxplot()\n",
    "        + scale_y_log10(labels = lambda arr: [\"{:,}\".format(x) for x in arr])\n",
    "    + theme(axis_text_x=element_text(angle=45, hjust=1))\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv-tidytuesday-py",
   "language": "python",
   "name": "venv-tidytuesday-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}