Add summarization section to analyzing benchmarks notebook.

PiperOrigin-RevId: 642637009
google · Jun 12, 2024 · 7d069fc · 7d069fc
1 parent 12d7c7c
commit 7d069fc
Showing 1 changed file with 41 additions and 7 deletions.
diff --git a/docs/guides/benchmarks/analyzing_benchmarks.ipynb b/docs/guides/benchmarks/analyzing_benchmarks.ipynb
@@ -47,7 +47,11 @@
         "from vizier import benchmarks as vzb\n",
         "from vizier.algorithms import designers\n",
         "from vizier.benchmarks import experimenters\n",
-        "from vizier.benchmarks import analyzers"
+        "from vizier.benchmarks import analyzers\n",
+        "\n",
+        "import itertools\n",
+        "import numpy as np\n",
+        "import pandas as pd"
       ]
     },
     {
@@ -71,10 +75,6 @@
       },
       "outputs": [],
       "source": [
-        "import itertools\n",
-        "import numpy as np\n",
-        "from vizier.benchmarks import experimenters\n",
-        "\n",
         "function_names = ['Sphere', 'Discus']\n",
         "dimensions = [4, 8]\n",
         "product_list = list(itertools.product(function_names, dimensions))\n",
@@ -149,8 +149,6 @@
       },
       "outputs": [],
       "source": [
-        "import pandas as pd\n",
-        "\n",
         "records_list = [\n",
         "    (rec.algorithm, dict(rec.experimenter_metadata), rec) for rec in records\n",
         "]\n",
@@ -351,6 +349,38 @@
         "analyzers.plot_from_records(analyzed_records_with_winrate)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BOlTT1OGGuyM"
+      },
+      "source": [
+        "## Summarizing with Comparators\n",
+        "\n",
+        "One of the main benefits of using `ConvergenceComparator` is that these metrics are usually already normalized across different optimization problems and contexts. We provide utility functions to perform summarization procedures across `BenchmarkRecords`, which can used with the `summarize` feature of the `BenchmarkRecordAnalyzer`.\n",
+        "\n",
+        "Note that we can write a custom score and summary function, but we use our default summarizer that simply plots the distribution of scores across problems as histograms.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7HrPNvSaHxzC"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "# Summarized across experimenters by giving reduced keys.\n",
+        "def record_to_reduced_keys(record):\n",
+        "  bbob_dict = json.loads(record.experimenter_metadata['bbob_factory'])\n",
+        "  experimenter = bbob_dict.pop('name')\n",
+        "  return experimenter, json.dumps(bbob_dict)\n",
+        "\n",
+        "summarized_records = analyzers.BenchmarkRecordAnalyzer.summarize(analyzed_records_with_winrate, record_to_reduced_keys)\n",
+        "analyzers.plot_from_records(summarized_records)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -366,6 +396,10 @@
   ],
   "metadata": {
     "colab": {
+      "last_runtime": {
+        "build_target": "//learning/vizier/service/colab:notebook",
+        "kind": "private"
+      },
       "name": "Analyzing Benchmarks.ipynb",
       "private_outputs": true,
       "provenance": []