Python2vs3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import boost_histogram as bh\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "skhep = pd.read_csv(\"scikit-hep-20180101-20250101.csv\", usecols=[\"timestamp\", \"file_project\", \"details_python\"], parse_dates=[\"timestamp\"], dtype={\"file_project\": \"category\", \"details_python\": \"category\"})\n",
    "skhep.dropna(inplace=True) # Drop 11 NA's for Python version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classic method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_pandas(skhep, *projects: str, year: int):\n",
    "    start=f\"{year}-01-01\"\n",
    "    end=f\"{year}-12-31\"\n",
    "    results = {}\n",
    "    for project in projects:\n",
    "        val = skhep[skhep.file_project == project]\n",
    "        val = val[(val.timestamp > start) & (val.timestamp < end)]\n",
    "        vers = val.details_python.str[0]\n",
    "        results[project] = (sum(vers == '2'), sum(vers == '3'))\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.83 s, sys: 283 ms, total: 3.11 s\n",
      "Wall time: 3.12 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "results2020 = compute_pandas(skhep, *set(skhep.file_project), year=2020)\n",
    "results2019 = compute_pandas(skhep, *set(skhep.file_project), year=2019)\n",
    "results2018 = compute_pandas(skhep, *set(skhep.file_project), year=2018)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Boost Histogram method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_bh(hist, *projects: str, year: int):\n",
    "    results = {}\n",
    "    for project in projects:\n",
    "        ver2 = hist[bh.loc(project), bh.loc(year), bh.loc(2)]\n",
    "        ver3 = hist[bh.loc(project), bh.loc(year), bh.loc(3)]\n",
    "        results[project] = (ver2, ver3)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist = bh.Histogram(\n",
    "    bh.axis.StrCategory(list(set(skhep.file_project))),\n",
    "    bh.axis.Integer(2018, 2021, underflow=False, overflow=False),\n",
    "    bh.axis.Integer(2, 4, underflow=False, overflow=False),\n",
    "    storage=bh.storage.Int64()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.57 s, sys: 186 ms, total: 1.76 s\n",
      "Wall time: 1.76 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = hist.fill(np.asarray(skhep.file_project, dtype=str),\n",
    "          np.asarray(skhep.timestamp.dt.year),\n",
    "          np.asarray(skhep.details_python.str[0].astype(int)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 441 ms, sys: 34.1 ms, total: 475 ms\n",
      "Wall time: 474 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "results2020 = compute_bh(hist, *set(skhep.file_project), year=2020)\n",
    "results2019 = compute_bh(hist, *set(skhep.file_project), year=2019)\n",
    "results2018 = compute_bh(hist, *set(skhep.file_project), year=2018)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make table (from either result set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show(*args):\n",
    "    results = args[0]\n",
    "    keys = filter(lambda x: sum(results[x]) > 100, results)\n",
    "    keys = sorted(keys, key=lambda x: results[x][0]/sum(results[x]))\n",
    "    for project in keys:\n",
    "        print(f\"| {project:20} |\", end=\"\")\n",
    "        for extra in args:\n",
    "            res = \"\"\n",
    "            if project in extra:\n",
    "                v2, v3 = extra[project]\n",
    "                tot = (v2 + v3) / 1000\n",
    "                if tot >= 0.1:\n",
    "                    frac = v2 / (v2 + v3)\n",
    "                    x = 1 if tot < 10 else 0\n",
    "                    y = 1 if frac < 0.1 else 0\n",
    "                    res = f\"{frac:5.{y}%} of {tot:4.{x}f}k\"\n",
    "            print(f\" {res:14} |\", end=\"\")\n",
    "        print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Package name         | 2020 (partial) | 2019           | 2018           |\n",
      "|----------------------|----------------|----------------|----------------|\n",
      "| scikit-optimize      |  0.2% of 2812k |  2.5% of 1349k |   13% of   85k |\n",
      "| mplhep               |  0.2% of  111k |  6.2% of  4.7k |                |\n",
      "| formulate            |  0.9% of  1.2k |   18% of  0.5k |                |\n",
      "| pyhf                 |  1.1% of  6.8k |   13% of  1.6k |                |\n",
      "| scikit-hep-testdata  |  1.4% of  4.6k |  2.4% of  3.5k |                |\n",
      "| pyjet                |  1.5% of  4.6k |   23% of  2.1k |   44% of  0.2k |\n",
      "| vegascope            |  2.8% of  0.7k |  1.3% of  5.7k |   48% of  0.1k |\n",
      "| awkward1             |  2.8% of  4.4k |   15% of  0.6k |                |\n",
      "| reana-client         |  4.0% of  1.4k |   42% of  0.2k |   48% of  0.1k |\n",
      "| aghast               |  6.3% of  0.1k |   26% of  0.4k |                |\n",
      "| boost-histogram      |  7.4% of  1.3k |   19% of  0.3k |                |\n",
      "| iminuit              |  8.0% of   48k |   16% of   60k |   34% of   10k |\n",
      "| probfit              |   11% of  1.0k |   11% of  2.0k |   35% of  0.5k |\n",
      "| scikit-hep           |   12% of  1.8k |   31% of  5.5k |   43% of  0.5k |\n",
      "| particle             |   13% of  2.2k |   31% of  3.9k |   26% of  0.1k |\n",
      "| numpythia            |   14% of  0.2k |   16% of  0.6k |   23% of  0.3k |\n",
      "| hepunits             |   16% of  5.9k |   30% of   12k |                |\n",
      "| uproot               |   16% of  205k |   17% of  198k |   55% of   14k |\n",
      "| uproot-methods       |   17% of  206k |   24% of  225k |   58% of   13k |\n",
      "| awkward              |   17% of  207k |   26% of  239k |   56% of   17k |\n",
      "| decaylanguage        |   22% of  0.2k |   25% of  0.3k |                |\n",
      "| rootpy               |   29% of  1.8k |   48% of  5.7k |   71% of  4.7k |\n",
      "| pylhe                |   44% of  0.1k |   62% of  0.1k |                |\n",
      "| root-numpy           |   54% of  4.6k |   63% of   12k |   70% of  7.9k |\n",
      "| madminer             |   57% of  0.3k |   62% of  1.1k |                |\n",
      "| yadage               |   67% of  0.9k |   72% of  1.7k |   89% of  0.7k |\n",
      "| root-pandas          |   71% of  3.3k |   61% of  4.4k |   96% of   24k |\n"
     ]
    }
   ],
   "source": [
    "print(\"| Package name         | 2020 (partial) | 2019           | 2018           |\")\n",
    "print(\"|----------------------|----------------|----------------|----------------|\")\n",
    "show(results2020, results2019, results2018)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:scikit-hep-tutorials]",
   "language": "python",
   "name": "conda-env-scikit-hep-tutorials-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}