Benchmark/PlotBenchmarkings.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Genometric/MSPC/blob/dev/Benchmark/PlotBenchmarkings.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "l49WL23ePSR_"
      },
      "outputs": [],
      "source": [
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import urllib.request\n",
        "import seaborn as sns\n",
        "\n",
        "sns.set_theme()\n",
        "sns.set_context(\"paper\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download benchmarking data from OSF."
      ],
      "metadata": {
        "id": "DikOuuxzA2FU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "benchmarking_results_filename = \"benchmarking_results.tsv\"\n",
        "urllib.request.urlretrieve(\"https://osf.io/download/jf7d5/\", benchmarking_results_filename)\n",
        "\n",
        "reps_benchmarking_filename = \"benchmarking_results_replicates.tsv\"\n",
        "urllib.request.urlretrieve(\"https://osf.io/download/mfvt3/\", reps_benchmarking_filename)"
      ],
      "metadata": {
        "id": "UDYD7Q4LA1nQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(benchmarking_results_filename, sep=\"\\t\")\n",
        "df_replicates = pd.read_csv(reps_benchmarking_filename, sep=\"\\t\")"
      ],
      "metadata": {
        "id": "tUrxUzhUPx7y"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "D0KRe-u_Q12e",
        "outputId": "ba692b73-e32a-4822-a863-0df41567dad6"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    mspc_version experiment_id  replicate_count  interval_count  \\\n",
              "0           v2.1   ENCSR000BNU                2           18573   \n",
              "1           v2.1   ENCSR000EFR                2          600000   \n",
              "2           v2.1   ENCSR000EGD                2           10834   \n",
              "3           v2.1   ENCSR000EGJ                2          600000   \n",
              "4           v2.1   ENCSR000EGL                2          600000   \n",
              "..           ...           ...              ...             ...   \n",
              "139       v6.0.0   ENCSR908CMW                2          600000   \n",
              "140       v6.0.0   ENCSR914NEI                2           44805   \n",
              "141       v6.0.0   ENCSR931HNY                2          600000   \n",
              "142       v6.0.0   ENCSR987PBI                2            1667   \n",
              "143       v6.0.0   ENCSR998AJK                2           40753   \n",
              "\n",
              "     runtime_seconds  peak_physical_memory_usage_bytes  \\\n",
              "0           1.008059                          44933120   \n",
              "1           3.501384                          72560640   \n",
              "2           0.794680                          43245568   \n",
              "3           3.544179                          72511488   \n",
              "4           3.243759                          73588736   \n",
              "..               ...                               ...   \n",
              "139         9.920540                         215388160   \n",
              "140         3.022518                         103391232   \n",
              "141         9.532760                         215715840   \n",
              "142         0.652432                          65130496   \n",
              "143         2.186346                         100589568   \n",
              "\n",
              "     peak_paged_memory_usage_bytes  peak_virtual_memory_usage_bytes  \n",
              "0                         37289984                        183336960  \n",
              "1                         65654784                        217939968  \n",
              "2                         35016704                        165838848  \n",
              "3                         66162688                        217939968  \n",
              "4                         67121152                        217939968  \n",
              "..                             ...                              ...  \n",
              "139                      181923840                    2204071256064  \n",
              "140                       68603904                    2204072173568  \n",
              "141                      181911552                    2204071256064  \n",
              "142                       27033600                    2204071370752  \n",
              "143                       66187264                    2204072828928  \n",
              "\n",
              "[144 rows x 8 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0673c04c-c268-4237-832d-1bce9fa72112\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mspc_version</th>\n",
              "      <th>experiment_id</th>\n",
              "      <th>replicate_count</th>\n",
              "      <th>interval_count</th>\n",
              "      <th>runtime_seconds</th>\n",
              "      <th>peak_physical_memory_usage_bytes</th>\n",
              "      <th>peak_paged_memory_usage_bytes</th>\n",
              "      <th>peak_virtual_memory_usage_bytes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>v2.1</td>\n",
              "      <td>ENCSR000BNU</td>\n",
              "      <td>2</td>\n",
              "      <td>18573</td>\n",
              "      <td>1.008059</td>\n",
              "      <td>44933120</td>\n",
              "      <td>37289984</td>\n",
              "      <td>183336960</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>v2.1</td>\n",
              "      <td>ENCSR000EFR</td>\n",
              "      <td>2</td>\n",
              "      <td>600000</td>\n",
              "      <td>3.501384</td>\n",
              "      <td>72560640</td>\n",
              "      <td>65654784</td>\n",
              "      <td>217939968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>v2.1</td>\n",
              "      <td>ENCSR000EGD</td>\n",
              "      <td>2</td>\n",
              "      <td>10834</td>\n",
              "      <td>0.794680</td>\n",
              "      <td>43245568</td>\n",
              "      <td>35016704</td>\n",
              "      <td>165838848</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>v2.1</td>\n",
              "      <td>ENCSR000EGJ</td>\n",
              "      <td>2</td>\n",
              "      <td>600000</td>\n",
              "      <td>3.544179</td>\n",
              "      <td>72511488</td>\n",
              "      <td>66162688</td>\n",
              "      <td>217939968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>v2.1</td>\n",
              "      <td>ENCSR000EGL</td>\n",
              "      <td>2</td>\n",
              "      <td>600000</td>\n",
              "      <td>3.243759</td>\n",
              "      <td>73588736</td>\n",
              "      <td>67121152</td>\n",
              "      <td>217939968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>139</th>\n",
              "      <td>v6.0.0</td>\n",
              "      <td>ENCSR908CMW</td>\n",
              "      <td>2</td>\n",
              "      <td>600000</td>\n",
              "      <td>9.920540</td>\n",
              "      <td>215388160</td>\n",
              "      <td>181923840</td>\n",
              "      <td>2204071256064</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>140</th>\n",
              "      <td>v6.0.0</td>\n",
              "      <td>ENCSR914NEI</td>\n",
              "      <td>2</td>\n",
              "      <td>44805</td>\n",
              "      <td>3.022518</td>\n",
              "      <td>103391232</td>\n",
              "      <td>68603904</td>\n",
              "      <td>2204072173568</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>141</th>\n",
              "      <td>v6.0.0</td>\n",
              "      <td>ENCSR931HNY</td>\n",
              "      <td>2</td>\n",
              "      <td>600000</td>\n",
              "      <td>9.532760</td>\n",
              "      <td>215715840</td>\n",
              "      <td>181911552</td>\n",
              "      <td>2204071256064</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142</th>\n",
              "      <td>v6.0.0</td>\n",
              "      <td>ENCSR987PBI</td>\n",
              "      <td>2</td>\n",
              "      <td>1667</td>\n",
              "      <td>0.652432</td>\n",
              "      <td>65130496</td>\n",
              "      <td>27033600</td>\n",
              "      <td>2204071370752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>143</th>\n",
              "      <td>v6.0.0</td>\n",
              "      <td>ENCSR998AJK</td>\n",
              "      <td>2</td>\n",
              "      <td>40753</td>\n",
              "      <td>2.186346</td>\n",
              "      <td>100589568</td>\n",
              "      <td>66187264</td>\n",
              "      <td>2204072828928</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>144 rows × 8 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0673c04c-c268-4237-832d-1bce9fa72112')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0673c04c-c268-4237-832d-1bce9fa72112 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0673c04c-c268-4237-832d-1bce9fa72112');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "versions_groups = df.groupby(\"mspc_version\")\n",
        "len(versions_groups)"
      ],
      "metadata": {
        "id": "2N53Cs8VwNUa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "releases = {}\n",
        "for release in versions_groups.groups:\n",
        "  g = versions_groups.get_group(release)\n",
        "  g_ = g.copy()\n",
        "  g_ = g_.drop('mspc_version', axis=1)\n",
        "  releases[release] = g_"
      ],
      "metadata": {
        "id": "DJCs4eoXADNU"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_increase_percentage(cur_val, pre_val):\n",
        "  return ((pre_val - cur_val) / abs(pre_val)) * 100\n",
        "\n",
        "def get_improvement_foldchange(cur_val, pre_val):\n",
        "  return pre_val/cur_val"
      ],
      "metadata": {
        "id": "bnVeeZOvqtUF"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "changes = []\n",
        "odf = releases[\"v6.0.0\"]\n",
        "for idx, row in releases[\"v1.1\"].iterrows():\n",
        "  other = odf.loc[(odf[\"interval_count\"] == row[\"interval_count\"]) & (odf[\"replicate_count\"] == row[\"replicate_count\"])]\n",
        "  if len(other) == 1:\n",
        "    changes.append({\n",
        "        \"experiment_id\": row[\"experiment_id\"],\n",
        "        \"replicate_count\": row[\"replicate_count\"],\n",
        "        \"interval_count\": row[\"interval_count\"],\n",
        "        \"runtime_change_percent\": get_improvement_foldchange(other[\"runtime_seconds\"].to_list()[0], row[\"runtime_seconds\"]),\n",
        "        \"peak_physical_memory_usage_change_percent\": get_improvement_foldchange(other[\"peak_physical_memory_usage_bytes\"].to_list()[0], row[\"peak_physical_memory_usage_bytes\"]),\n",
        "        \"peak_paged_memory_usage_change_percent\": get_improvement_foldchange(other[\"peak_paged_memory_usage_bytes\"].to_list()[0], row[\"peak_paged_memory_usage_bytes\"]),\n",
        "        \"peak_virtual_memory_usage_change_percent\": get_improvement_foldchange(other[\"peak_virtual_memory_usage_bytes\"].to_list()[0], row[\"peak_virtual_memory_usage_bytes\"])\n",
        "    })\n",
        "  elif len(other) == 0:\n",
        "    print(\"Could not find a match; interval count \" + str(row[\"interval_count\"]) + \", and replicate count \"+ str(row[\"replicate_count\"]))\n",
        "    continue\n",
        "  else:\n",
        "    print(\"More than one match found; interval count \" + str(row[\"interval_count\"]) + \", and replicate count \" + str(row[\"replicate_count\"]))\n",
        "    continue"
      ],
      "metadata": {
        "id": "ES10HnjqppDb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(changes)"
      ],
      "metadata": {
        "id": "Ixnkx358s4Q-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "changes_df = pd.DataFrame(changes)"
      ],
      "metadata": {
        "id": "UgFx7rAsvuQM"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots(1, 1, figsize=(6, 4))\n",
        "fig.set_dpi(600)\n",
        "\n",
        "\n",
        "changes_df.sort_values(\"runtime_change_percent\", ascending=False, inplace=True)\n",
        "sns.barplot(data=changes_df, x=\"interval_count\", y=\"runtime_change_percent\", order=changes_df[\"interval_count\"], facecolor=(0.2, 0.2, 0.4, 1), ax=ax)\n",
        "ax.set(xlabel=\"Interval count\", ylabel=\"Runtime Improvement fold change\\n\")\n",
        "#ax.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',') + \"x\"))\n",
        "ax.set_yscale(\"log\", basey=2)\n",
        "ax.set_xticklabels(ax.get_xticklabels(), rotation=90)\n",
        "ax.title.set_text(\"Runtime improvement between\\nMSPC v1 and v6\")\n",
        "\n",
        "plt.tight_layout()\n",
        "plt.savefig('benchmarks.png')"
      ],
      "metadata": {
        "id": "Mh1HFs0SJlGW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Convert peak physical memory usage scale from bytes to mega bytes."
      ],
      "metadata": {
        "id": "kaYUM-_H-YD6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "peak_mem_col_name = \"peak_physical_memory_usage_bytes\"\n",
        "df_replicates[peak_mem_col_name] = df_replicates[peak_mem_col_name].div(1024*1024)"
      ],
      "metadata": {
        "id": "4a9j-Gt6-XHx"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def hide_even_num_ticks(axis):\n",
        "  [l.set_visible(False) for (i,l) in enumerate(axis.xaxis.get_ticklabels()) if i % 2 != 0]"
      ],
      "metadata": {
        "id": "XCgtq0MCBMhA"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots(1,2, figsize=(10,4))\n",
        "fig.set_dpi(600)\n",
        "\n",
        "sns.boxplot(x=df_replicates[\"replicate_count\"], y=df_replicates[\"runtime_seconds\"], ax=ax[0])\n",
        "ax[0].set(xlabel=\"\\n\\nReplicate count\", ylabel=\"Runtime (seconds)\")\n",
        "ax[0].title.set_text(\"Changes in runtime\\nw.r.t increase in the replicate count\")\n",
        "\n",
        "sns.boxplot(x=df_replicates[\"replicate_count\"], y=df_replicates[peak_mem_col_name], ax=ax[1])\n",
        "ax[1].set(xlabel=\"\\n\\nReplicate count\", ylabel=\"Peak Physical Memory Usage\\n(Megabyte)\")\n",
        "ax[1].title.set_text(\"\\nChanges in memory requirement\\nw.r.t increase in the replicate count\")\n",
        "\n",
        "hide_even_num_ticks(ax[0])\n",
        "hide_even_num_ticks(ax[1])\n",
        "\n",
        "# Set color of boxes in the boxplot.\n",
        "# for i,box in enumerate(ax[1].artists):\n",
        "#     box.set_edgecolor('black')\n",
        "#     box.set_facecolor((0.0, 0.0, 0.0, 0))\n",
        "#     # iterate over whiskers and median lines\n",
        "#     for j in range(6 * i, 6 * (i + 1)):\n",
        "#          ax[1].lines[j].set_color('black')\n",
        "\n",
        "plt.tight_layout()\n",
        "plt.savefig('scalability.png')"
      ],
      "metadata": {
        "id": "EnXbmt8F653Q"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}