effectpower.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# ==========================================\n",
        "\n",
        "# Author: Leonhard Felix Richter\n",
        "# Date:   25 Jan 2023\n",
        "# =========================================="
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 319,
      "metadata": {
        "notebookRunGroups": {
          "groupValue": "1"
        }
      },
      "outputs": [],
      "source": [
        "import math\n",
        "import numpy as np\n",
        "from scipy.special import gamma\n",
        "import scipy.stats as statistics\n",
        "import pandas as pd\n",
        "import swifter\n",
        "from typing import Literal\n",
        "import pickle"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "notebookRunGroups": {
          "groupValue": "1"
        }
      },
      "outputs": [],
      "source": [
        "with open('results/data_gd_cut.p', 'rb') as f:\n",
        "    data = pickle.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "notebookRunGroups": {
          "groupValue": "1"
        }
      },
      "outputs": [],
      "source": [
        "with open(\"results/stats_gd_cut.p\",'rb') as file:\n",
        "    stats = pickle.load(file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>instance</th>\n",
              "      <th>tdvp</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th>i</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>0</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcbc7407910&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d05540&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>0</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcbc7407910&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d05540&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <th>1</th>\n",
              "      <th>0</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d05720&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">5</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>5</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d07430&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d07730&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d07a30&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d07d30&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52af4070&gt;</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>instance                                    &lt;M...</td>\n",
              "      <td>algorithm                                     ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>74 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                       instance  \\\n",
              "n p i                                             \n",
              "3 1 0  <MaxCut.MaxCut object at 0x7fcbc7407910>   \n",
              "    1  <MaxCut.MaxCut object at 0x7fcb52d05540>   \n",
              "  2 0  <MaxCut.MaxCut object at 0x7fcbc7407910>   \n",
              "    1  <MaxCut.MaxCut object at 0x7fcb52d05540>   \n",
              "4 1 0  <MaxCut.MaxCut object at 0x7fcb52d05720>   \n",
              "...                                         ...   \n",
              "5 4 5  <MaxCut.MaxCut object at 0x7fcb52d07430>   \n",
              "    6  <MaxCut.MaxCut object at 0x7fcb52d07730>   \n",
              "    7  <MaxCut.MaxCut object at 0x7fcb52d07a30>   \n",
              "    8  <MaxCut.MaxCut object at 0x7fcb52d07d30>   \n",
              "    9  <MaxCut.MaxCut object at 0x7fcb52af4070>   \n",
              "\n",
              "                                                    tdvp  \\\n",
              "n p i                                                      \n",
              "3 1 0  instance                                    <M...   \n",
              "    1  instance                                    <M...   \n",
              "  2 0  instance                                    <M...   \n",
              "    1  instance                                    <M...   \n",
              "4 1 0  instance                                    <M...   \n",
              "...                                                  ...   \n",
              "5 4 5  instance                                    <M...   \n",
              "    6  instance                                    <M...   \n",
              "    7  instance                                    <M...   \n",
              "    8  instance                                    <M...   \n",
              "    9  instance                                    <M...   \n",
              "\n",
              "                                                   scipy  \\\n",
              "n p i                                                      \n",
              "3 1 0  instance                                    <M...   \n",
              "    1  instance                                    <M...   \n",
              "  2 0  instance                                    <M...   \n",
              "    1  instance                                    <M...   \n",
              "4 1 0  instance                                    <M...   \n",
              "...                                                  ...   \n",
              "5 4 5  instance                                    <M...   \n",
              "    6  instance                                    <M...   \n",
              "    7  instance                                    <M...   \n",
              "    8  instance                                    <M...   \n",
              "    9  instance                                    <M...   \n",
              "\n",
              "                                        gradient_descent  \n",
              "n p i                                                     \n",
              "3 1 0  algorithm                                     ...  \n",
              "    1  algorithm                                     ...  \n",
              "  2 0  algorithm                                     ...  \n",
              "    1  algorithm                                     ...  \n",
              "4 1 0  algorithm                                     ...  \n",
              "...                                                  ...  \n",
              "5 4 5  algorithm                                     ...  \n",
              "    6  algorithm                                     ...  \n",
              "    7  algorithm                                     ...  \n",
              "    8  algorithm                                     ...  \n",
              "    9  algorithm                                     ...  \n",
              "\n",
              "[74 rows x 4 columns]"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>instance</th>\n",
              "      <th>p</th>\n",
              "      <th>n</th>\n",
              "      <th>delta_0</th>\n",
              "      <th>tollarance</th>\n",
              "      <th>algorithm</th>\n",
              "      <th>res</th>\n",
              "      <th>delta</th>\n",
              "      <th>value</th>\n",
              "      <th>path</th>\n",
              "      <th>...</th>\n",
              "      <th>answer</th>\n",
              "      <th>bin_answer</th>\n",
              "      <th>best_cost_value</th>\n",
              "      <th>approximation ratio</th>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <th>circuit count</th>\n",
              "      <th>gate count</th>\n",
              "      <th>path length</th>\n",
              "      <th>groundspace overlap</th>\n",
              "      <th>groundspace sharpness</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>i</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d05720&gt;</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "      <td>(1, 1, 1, 1, 1, 1, 1, 1, 1, 1)</td>\n",
              "      <td>0.01</td>\n",
              "      <td>tdvp</td>\n",
              "      <td>\\n        None terminated with no sucess with ...</td>\n",
              "      <td>(1, 1, 1, 1, 1, 1, 1, 1, 1, 1)</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d05d50&gt;</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "      <td>(1, 1, 1, 1, 1, 1, 1, 1, 1, 1)</td>\n",
              "      <td>0.01</td>\n",
              "      <td>tdvp</td>\n",
              "      <td>\\n        None terminated with no sucess with ...</td>\n",
              "      <td>(1, 1, 1, 1, 1, 1, 1, 1, 1, 1)</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>&lt;MaxCut.MaxCut object at 0x7fcb52d06290&gt;</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "      <td>(1, 1, 1, 1, 1, 1, 1, 1, 1, 1)</td>\n",
              "      <td>0.01</td>\n",
              "      <td>tdvp</td>\n",
              "      <td>\\n        None terminated with no sucess with ...</td>\n",
              "      <td>(1, 1, 1, 1, 1, 1, 1, 1, 1, 1)</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>3 rows × 25 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                   instance  p  n  \\\n",
              "i                                                   \n",
              "0  <MaxCut.MaxCut object at 0x7fcb52d05720>  5  4   \n",
              "3  <MaxCut.MaxCut object at 0x7fcb52d05d50>  5  4   \n",
              "5  <MaxCut.MaxCut object at 0x7fcb52d06290>  5  4   \n",
              "\n",
              "                          delta_0 tollarance algorithm  \\\n",
              "i                                                        \n",
              "0  (1, 1, 1, 1, 1, 1, 1, 1, 1, 1)       0.01      tdvp   \n",
              "3  (1, 1, 1, 1, 1, 1, 1, 1, 1, 1)       0.01      tdvp   \n",
              "5  (1, 1, 1, 1, 1, 1, 1, 1, 1, 1)       0.01      tdvp   \n",
              "\n",
              "                                                 res  \\\n",
              "i                                                      \n",
              "0  \\n        None terminated with no sucess with ...   \n",
              "3  \\n        None terminated with no sucess with ...   \n",
              "5  \\n        None terminated with no sucess with ...   \n",
              "\n",
              "                            delta value  path  ... answer bin_answer  \\\n",
              "i                                              ...                     \n",
              "0  (1, 1, 1, 1, 1, 1, 1, 1, 1, 1)  None  None  ...    NaN        NaN   \n",
              "3  (1, 1, 1, 1, 1, 1, 1, 1, 1, 1)  None  None  ...    NaN        NaN   \n",
              "5  (1, 1, 1, 1, 1, 1, 1, 1, 1, 1)  None  None  ...    NaN        NaN   \n",
              "\n",
              "  best_cost_value approximation ratio expectation approximation ratio  \\\n",
              "i                                                                       \n",
              "0             NaN                 NaN                             NaN   \n",
              "3             NaN                 NaN                             NaN   \n",
              "5             NaN                 NaN                             NaN   \n",
              "\n",
              "  circuit count gate count path length groundspace overlap  \\\n",
              "i                                                            \n",
              "0           NaN        NaN         NaN                 NaN   \n",
              "3           NaN        NaN         NaN                 NaN   \n",
              "5           NaN        NaN         NaN                 NaN   \n",
              "\n",
              "  groundspace sharpness  \n",
              "i                        \n",
              "0                   NaN  \n",
              "3                   NaN  \n",
              "5                   NaN  \n",
              "\n",
              "[3 rows x 25 columns]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data_col = pd.DataFrame(index=data['tdvp'].xs((4,5), level=[0,1]).index, columns=data['tdvp'].loc[4,5,0].index)\n",
        "for i in data_col.index:\n",
        "    data_col.loc[i] = data['tdvp'].xs((4,5), level=[0,1]).loc[i]\n",
        "data_col[data_col['approximation ratio'].apply(lambda x: np.isnan(x))]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [],
      "source": [
        "relevant_count = pd.DataFrame(index=stats.index, columns=pd.MultiIndex.from_product([['relevance'],['count', 'total', 'ratio']]))\n",
        "\n",
        "def count_relevance(col):\n",
        "    n,alg,p = idx = col.name\n",
        "    df = data[alg].xs((n,p), level=[0,1])\n",
        "    data_col = pd.DataFrame(index=df.index, columns=data[alg].loc[n,p,0].index)\n",
        "    for i in data_col.index:\n",
        "        data_col.loc[i] = df.loc[i]\n",
        "    col['count'] = len(data_col[data_col['answer'].apply(lambda x: not np.isnan(x))])\n",
        "    col['total'] = len(data_col)\n",
        "    col['ratio'] = col['count'] / col['total']\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"3\" halign=\"left\">relevance</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>total</th>\n",
              "      <th>ratio</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>Algorithm</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"6\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"15\" valign=\"top\">4</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>0.833333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>0.666667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>3</td>\n",
              "      <td>6</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"12\" valign=\"top\">5</th>\n",
              "      <th rowspan=\"4\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>9</td>\n",
              "      <td>10</td>\n",
              "      <td>0.9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7</td>\n",
              "      <td>10</td>\n",
              "      <td>0.7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6</td>\n",
              "      <td>10</td>\n",
              "      <td>0.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                     relevance                \n",
              "                         count total     ratio\n",
              "n Algorithm        p                          \n",
              "3 tdvp             1         2     2       1.0\n",
              "                   2         2     2       1.0\n",
              "  scipy            1         2     2       1.0\n",
              "                   2         2     2       1.0\n",
              "  gradient_descent 1         2     2       1.0\n",
              "                   2         2     2       1.0\n",
              "4 tdvp             1         6     6       1.0\n",
              "                   2         6     6       1.0\n",
              "                   3         5     6  0.833333\n",
              "                   4         4     6  0.666667\n",
              "                   5         3     6       0.5\n",
              "  scipy            1         6     6       1.0\n",
              "                   2         6     6       1.0\n",
              "                   3         6     6       1.0\n",
              "                   4         6     6       1.0\n",
              "                   5         6     6       1.0\n",
              "  gradient_descent 1         6     6       1.0\n",
              "                   2         6     6       1.0\n",
              "                   3         6     6       1.0\n",
              "                   4         6     6       1.0\n",
              "                   5         6     6       1.0\n",
              "5 tdvp             1        10    10       1.0\n",
              "                   2         9    10       0.9\n",
              "                   3         7    10       0.7\n",
              "                   4         6    10       0.6\n",
              "  scipy            1        10    10       1.0\n",
              "                   2        10    10       1.0\n",
              "                   3        10    10       1.0\n",
              "                   4        10    10       1.0\n",
              "  gradient_descent 1        10    10       1.0\n",
              "                   2        10    10       1.0\n",
              "                   3        10    10       1.0\n",
              "                   4        10    10       1.0"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "relevant_count['relevance'].apply(count_relevance, axis=1)\n",
        "relevant_count"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/tmp/ipykernel_94938/2346214742.py:1: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
            "  stats = stats.drop(columns=['relevance']).join(relevant_count)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"4\" halign=\"left\">expectation approximation ratio</th>\n",
              "      <th colspan=\"4\" halign=\"left\">groundspace overlap</th>\n",
              "      <th colspan=\"2\" halign=\"left\">groundspace sharpness</th>\n",
              "      <th>...</th>\n",
              "      <th>steps</th>\n",
              "      <th colspan=\"4\" halign=\"left\">path length</th>\n",
              "      <th colspan=\"2\" halign=\"left\">expectation approximation ratio per circuits</th>\n",
              "      <th colspan=\"3\" halign=\"left\">relevance</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>...</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>count</th>\n",
              "      <th>total</th>\n",
              "      <th>ratio</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>Algorithm</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>0.911088</td>\n",
              "      <td>0.073714</td>\n",
              "      <td>0.858965</td>\n",
              "      <td>0.963212</td>\n",
              "      <td>0.869012</td>\n",
              "      <td>0.145009</td>\n",
              "      <td>0.766475</td>\n",
              "      <td>0.971549</td>\n",
              "      <td>0.294665</td>\n",
              "      <td>0.181596</td>\n",
              "      <td>...</td>\n",
              "      <td>87.0</td>\n",
              "      <td>0.912754</td>\n",
              "      <td>0.036438</td>\n",
              "      <td>0.886988</td>\n",
              "      <td>0.938520</td>\n",
              "      <td>0.000309</td>\n",
              "      <td>-0.000002</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.930390</td>\n",
              "      <td>0.063845</td>\n",
              "      <td>0.885245</td>\n",
              "      <td>0.975536</td>\n",
              "      <td>0.931829</td>\n",
              "      <td>0.071453</td>\n",
              "      <td>0.881304</td>\n",
              "      <td>0.982354</td>\n",
              "      <td>0.227546</td>\n",
              "      <td>0.135601</td>\n",
              "      <td>...</td>\n",
              "      <td>147.0</td>\n",
              "      <td>0.823661</td>\n",
              "      <td>0.014654</td>\n",
              "      <td>0.813299</td>\n",
              "      <td>0.834024</td>\n",
              "      <td>0.000069</td>\n",
              "      <td>-0.000013</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>0.778976</td>\n",
              "      <td>0.112656</td>\n",
              "      <td>0.699316</td>\n",
              "      <td>0.858635</td>\n",
              "      <td>0.766332</td>\n",
              "      <td>0.006235</td>\n",
              "      <td>0.761923</td>\n",
              "      <td>0.770741</td>\n",
              "      <td>0.423131</td>\n",
              "      <td>0.003925</td>\n",
              "      <td>...</td>\n",
              "      <td>24.0</td>\n",
              "      <td>0.131371</td>\n",
              "      <td>0.094648</td>\n",
              "      <td>0.064445</td>\n",
              "      <td>0.198298</td>\n",
              "      <td>0.031795</td>\n",
              "      <td>0.003681</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.930466</td>\n",
              "      <td>0.058679</td>\n",
              "      <td>0.888974</td>\n",
              "      <td>0.971958</td>\n",
              "      <td>0.893380</td>\n",
              "      <td>0.117401</td>\n",
              "      <td>0.810365</td>\n",
              "      <td>0.976395</td>\n",
              "      <td>0.271914</td>\n",
              "      <td>0.169845</td>\n",
              "      <td>...</td>\n",
              "      <td>50.0</td>\n",
              "      <td>0.081390</td>\n",
              "      <td>0.084589</td>\n",
              "      <td>0.021576</td>\n",
              "      <td>0.141203</td>\n",
              "      <td>0.018609</td>\n",
              "      <td>0.000647</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>0.306478</td>\n",
              "      <td>0.432672</td>\n",
              "      <td>0.000533</td>\n",
              "      <td>0.612424</td>\n",
              "      <td>0.358776</td>\n",
              "      <td>0.307027</td>\n",
              "      <td>0.141675</td>\n",
              "      <td>0.575877</td>\n",
              "      <td>0.421463</td>\n",
              "      <td>0.102879</td>\n",
              "      <td>...</td>\n",
              "      <td>59.0</td>\n",
              "      <td>0.230496</td>\n",
              "      <td>0.310428</td>\n",
              "      <td>0.010990</td>\n",
              "      <td>0.450001</td>\n",
              "      <td>0.001293</td>\n",
              "      <td>0.001826</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 41 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                     expectation approximation ratio                      \\\n",
              "                                                mean       std       min   \n",
              "n Algorithm        p                                                       \n",
              "3 tdvp             1                        0.911088  0.073714  0.858965   \n",
              "                   2                        0.930390  0.063845  0.885245   \n",
              "  scipy            1                        0.778976  0.112656  0.699316   \n",
              "                   2                        0.930466  0.058679  0.888974   \n",
              "  gradient_descent 1                        0.306478  0.432672  0.000533   \n",
              "\n",
              "                               groundspace overlap                      \\\n",
              "                           max                mean       std       min   \n",
              "n Algorithm        p                                                     \n",
              "3 tdvp             1  0.963212            0.869012  0.145009  0.766475   \n",
              "                   2  0.975536            0.931829  0.071453  0.881304   \n",
              "  scipy            1  0.858635            0.766332  0.006235  0.761923   \n",
              "                   2  0.971958            0.893380  0.117401  0.810365   \n",
              "  gradient_descent 1  0.612424            0.358776  0.307027  0.141675   \n",
              "\n",
              "                               groundspace sharpness            ...  steps  \\\n",
              "                           max                  mean       std  ...    max   \n",
              "n Algorithm        p                                            ...          \n",
              "3 tdvp             1  0.971549              0.294665  0.181596  ...   87.0   \n",
              "                   2  0.982354              0.227546  0.135601  ...  147.0   \n",
              "  scipy            1  0.770741              0.423131  0.003925  ...   24.0   \n",
              "                   2  0.976395              0.271914  0.169845  ...   50.0   \n",
              "  gradient_descent 1  0.575877              0.421463  0.102879  ...   59.0   \n",
              "\n",
              "                     path length                                \\\n",
              "                            mean       std       min       max   \n",
              "n Algorithm        p                                             \n",
              "3 tdvp             1    0.912754  0.036438  0.886988  0.938520   \n",
              "                   2    0.823661  0.014654  0.813299  0.834024   \n",
              "  scipy            1    0.131371  0.094648  0.064445  0.198298   \n",
              "                   2    0.081390  0.084589  0.021576  0.141203   \n",
              "  gradient_descent 1    0.230496  0.310428  0.010990  0.450001   \n",
              "\n",
              "                     expectation approximation ratio per circuits            \\\n",
              "                                                             mean       std   \n",
              "n Algorithm        p                                                          \n",
              "3 tdvp             1                                     0.000309 -0.000002   \n",
              "                   2                                     0.000069 -0.000013   \n",
              "  scipy            1                                     0.031795  0.003681   \n",
              "                   2                                     0.018609  0.000647   \n",
              "  gradient_descent 1                                     0.001293  0.001826   \n",
              "\n",
              "                     relevance              \n",
              "                         count total ratio  \n",
              "n Algorithm        p                        \n",
              "3 tdvp             1         2     2   1.0  \n",
              "                   2         2     2   1.0  \n",
              "  scipy            1         2     2   1.0  \n",
              "                   2         2     2   1.0  \n",
              "  gradient_descent 1         2     2   1.0  \n",
              "\n",
              "[5 rows x 41 columns]"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats = stats.drop(columns=['relevance']).join(relevant_count)\n",
        "stats.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {},
      "outputs": [],
      "source": [
        "with open(\"results/stats_gd_cut.p\",'wb') as file:\n",
        "    pickle.dump(stats,file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"4\" halign=\"left\">expectation approximation ratio</th>\n",
              "      <th colspan=\"4\" halign=\"left\">groundspace overlap</th>\n",
              "      <th colspan=\"2\" halign=\"left\">groundspace sharpness</th>\n",
              "      <th>...</th>\n",
              "      <th>steps</th>\n",
              "      <th colspan=\"4\" halign=\"left\">path length</th>\n",
              "      <th colspan=\"2\" halign=\"left\">expectation approximation ratio per circuits</th>\n",
              "      <th colspan=\"3\" halign=\"left\">relevance</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>...</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>success</th>\n",
              "      <th>total</th>\n",
              "      <th>ratio</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>Algorithm</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"6\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>0.911088</td>\n",
              "      <td>0.073714</td>\n",
              "      <td>0.858965</td>\n",
              "      <td>0.963212</td>\n",
              "      <td>0.869012</td>\n",
              "      <td>0.145009</td>\n",
              "      <td>0.766475</td>\n",
              "      <td>0.971549</td>\n",
              "      <td>0.294665</td>\n",
              "      <td>0.181596</td>\n",
              "      <td>...</td>\n",
              "      <td>87.0</td>\n",
              "      <td>0.912754</td>\n",
              "      <td>0.036438</td>\n",
              "      <td>0.886988</td>\n",
              "      <td>0.938520</td>\n",
              "      <td>0.000309</td>\n",
              "      <td>-0.000002</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.930390</td>\n",
              "      <td>0.063845</td>\n",
              "      <td>0.885245</td>\n",
              "      <td>0.975536</td>\n",
              "      <td>0.931829</td>\n",
              "      <td>0.071453</td>\n",
              "      <td>0.881304</td>\n",
              "      <td>0.982354</td>\n",
              "      <td>0.227546</td>\n",
              "      <td>0.135601</td>\n",
              "      <td>...</td>\n",
              "      <td>147.0</td>\n",
              "      <td>0.823661</td>\n",
              "      <td>0.014654</td>\n",
              "      <td>0.813299</td>\n",
              "      <td>0.834024</td>\n",
              "      <td>0.000069</td>\n",
              "      <td>-0.000013</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>0.778976</td>\n",
              "      <td>0.112656</td>\n",
              "      <td>0.699316</td>\n",
              "      <td>0.858635</td>\n",
              "      <td>0.766332</td>\n",
              "      <td>0.006235</td>\n",
              "      <td>0.761923</td>\n",
              "      <td>0.770741</td>\n",
              "      <td>0.423131</td>\n",
              "      <td>0.003925</td>\n",
              "      <td>...</td>\n",
              "      <td>24.0</td>\n",
              "      <td>0.131371</td>\n",
              "      <td>0.094648</td>\n",
              "      <td>0.064445</td>\n",
              "      <td>0.198298</td>\n",
              "      <td>0.031795</td>\n",
              "      <td>0.003681</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.930466</td>\n",
              "      <td>0.058679</td>\n",
              "      <td>0.888974</td>\n",
              "      <td>0.971958</td>\n",
              "      <td>0.893380</td>\n",
              "      <td>0.117401</td>\n",
              "      <td>0.810365</td>\n",
              "      <td>0.976395</td>\n",
              "      <td>0.271914</td>\n",
              "      <td>0.169845</td>\n",
              "      <td>...</td>\n",
              "      <td>50.0</td>\n",
              "      <td>0.081390</td>\n",
              "      <td>0.084589</td>\n",
              "      <td>0.021576</td>\n",
              "      <td>0.141203</td>\n",
              "      <td>0.018609</td>\n",
              "      <td>0.000647</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>0.306478</td>\n",
              "      <td>0.432672</td>\n",
              "      <td>0.000533</td>\n",
              "      <td>0.612424</td>\n",
              "      <td>0.358776</td>\n",
              "      <td>0.307027</td>\n",
              "      <td>0.141675</td>\n",
              "      <td>0.575877</td>\n",
              "      <td>0.421463</td>\n",
              "      <td>0.102879</td>\n",
              "      <td>...</td>\n",
              "      <td>59.0</td>\n",
              "      <td>0.230496</td>\n",
              "      <td>0.310428</td>\n",
              "      <td>0.010990</td>\n",
              "      <td>0.450001</td>\n",
              "      <td>0.001293</td>\n",
              "      <td>0.001826</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.652124</td>\n",
              "      <td>0.047679</td>\n",
              "      <td>0.618410</td>\n",
              "      <td>0.685838</td>\n",
              "      <td>0.636691</td>\n",
              "      <td>0.049783</td>\n",
              "      <td>0.601489</td>\n",
              "      <td>0.671892</td>\n",
              "      <td>0.479558</td>\n",
              "      <td>0.014190</td>\n",
              "      <td>...</td>\n",
              "      <td>95.0</td>\n",
              "      <td>0.088509</td>\n",
              "      <td>0.090446</td>\n",
              "      <td>0.024554</td>\n",
              "      <td>0.152464</td>\n",
              "      <td>0.000857</td>\n",
              "      <td>0.000063</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">4</th>\n",
              "      <th rowspan=\"4\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>0.555697</td>\n",
              "      <td>0.320149</td>\n",
              "      <td>0.133846</td>\n",
              "      <td>0.963225</td>\n",
              "      <td>0.464344</td>\n",
              "      <td>0.373130</td>\n",
              "      <td>0.019871</td>\n",
              "      <td>0.962394</td>\n",
              "      <td>0.331865</td>\n",
              "      <td>0.164583</td>\n",
              "      <td>...</td>\n",
              "      <td>82.0</td>\n",
              "      <td>0.837968</td>\n",
              "      <td>0.199199</td>\n",
              "      <td>0.434681</td>\n",
              "      <td>0.955152</td>\n",
              "      <td>0.000068</td>\n",
              "      <td>0.000033</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.914538</td>\n",
              "      <td>0.023949</td>\n",
              "      <td>0.873142</td>\n",
              "      <td>0.947408</td>\n",
              "      <td>0.895209</td>\n",
              "      <td>0.049664</td>\n",
              "      <td>0.831052</td>\n",
              "      <td>0.943499</td>\n",
              "      <td>0.297600</td>\n",
              "      <td>0.061860</td>\n",
              "      <td>...</td>\n",
              "      <td>701.0</td>\n",
              "      <td>0.782392</td>\n",
              "      <td>0.116077</td>\n",
              "      <td>0.602306</td>\n",
              "      <td>0.896681</td>\n",
              "      <td>0.000012</td>\n",
              "      <td>-0.000012</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.926812</td>\n",
              "      <td>0.027653</td>\n",
              "      <td>0.895857</td>\n",
              "      <td>0.966206</td>\n",
              "      <td>0.866436</td>\n",
              "      <td>0.068250</td>\n",
              "      <td>0.795922</td>\n",
              "      <td>0.967577</td>\n",
              "      <td>0.324697</td>\n",
              "      <td>0.090627</td>\n",
              "      <td>...</td>\n",
              "      <td>3000.0</td>\n",
              "      <td>0.593010</td>\n",
              "      <td>0.182262</td>\n",
              "      <td>0.372322</td>\n",
              "      <td>0.862888</td>\n",
              "      <td>0.000002</td>\n",
              "      <td>-0.000003</td>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>0.833333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.943432</td>\n",
              "      <td>0.049538</td>\n",
              "      <td>0.869999</td>\n",
              "      <td>0.974706</td>\n",
              "      <td>0.930693</td>\n",
              "      <td>0.062414</td>\n",
              "      <td>0.843536</td>\n",
              "      <td>0.977556</td>\n",
              "      <td>0.232359</td>\n",
              "      <td>0.100608</td>\n",
              "      <td>...</td>\n",
              "      <td>225.0</td>\n",
              "      <td>0.675338</td>\n",
              "      <td>0.099432</td>\n",
              "      <td>0.564874</td>\n",
              "      <td>0.806639</td>\n",
              "      <td>0.000006</td>\n",
              "      <td>-0.000001</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>0.666667</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10 rows × 41 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                     expectation approximation ratio                      \\\n",
              "                                                mean       std       min   \n",
              "n Algorithm        p                                                       \n",
              "3 tdvp             1                        0.911088  0.073714  0.858965   \n",
              "                   2                        0.930390  0.063845  0.885245   \n",
              "  scipy            1                        0.778976  0.112656  0.699316   \n",
              "                   2                        0.930466  0.058679  0.888974   \n",
              "  gradient_descent 1                        0.306478  0.432672  0.000533   \n",
              "                   2                        0.652124  0.047679  0.618410   \n",
              "4 tdvp             1                        0.555697  0.320149  0.133846   \n",
              "                   2                        0.914538  0.023949  0.873142   \n",
              "                   3                        0.926812  0.027653  0.895857   \n",
              "                   4                        0.943432  0.049538  0.869999   \n",
              "\n",
              "                               groundspace overlap                      \\\n",
              "                           max                mean       std       min   \n",
              "n Algorithm        p                                                     \n",
              "3 tdvp             1  0.963212            0.869012  0.145009  0.766475   \n",
              "                   2  0.975536            0.931829  0.071453  0.881304   \n",
              "  scipy            1  0.858635            0.766332  0.006235  0.761923   \n",
              "                   2  0.971958            0.893380  0.117401  0.810365   \n",
              "  gradient_descent 1  0.612424            0.358776  0.307027  0.141675   \n",
              "                   2  0.685838            0.636691  0.049783  0.601489   \n",
              "4 tdvp             1  0.963225            0.464344  0.373130  0.019871   \n",
              "                   2  0.947408            0.895209  0.049664  0.831052   \n",
              "                   3  0.966206            0.866436  0.068250  0.795922   \n",
              "                   4  0.974706            0.930693  0.062414  0.843536   \n",
              "\n",
              "                               groundspace sharpness            ...   steps  \\\n",
              "                           max                  mean       std  ...     max   \n",
              "n Algorithm        p                                            ...           \n",
              "3 tdvp             1  0.971549              0.294665  0.181596  ...    87.0   \n",
              "                   2  0.982354              0.227546  0.135601  ...   147.0   \n",
              "  scipy            1  0.770741              0.423131  0.003925  ...    24.0   \n",
              "                   2  0.976395              0.271914  0.169845  ...    50.0   \n",
              "  gradient_descent 1  0.575877              0.421463  0.102879  ...    59.0   \n",
              "                   2  0.671892              0.479558  0.014190  ...    95.0   \n",
              "4 tdvp             1  0.962394              0.331865  0.164583  ...    82.0   \n",
              "                   2  0.943499              0.297600  0.061860  ...   701.0   \n",
              "                   3  0.967577              0.324697  0.090627  ...  3000.0   \n",
              "                   4  0.977556              0.232359  0.100608  ...   225.0   \n",
              "\n",
              "                     path length                                \\\n",
              "                            mean       std       min       max   \n",
              "n Algorithm        p                                             \n",
              "3 tdvp             1    0.912754  0.036438  0.886988  0.938520   \n",
              "                   2    0.823661  0.014654  0.813299  0.834024   \n",
              "  scipy            1    0.131371  0.094648  0.064445  0.198298   \n",
              "                   2    0.081390  0.084589  0.021576  0.141203   \n",
              "  gradient_descent 1    0.230496  0.310428  0.010990  0.450001   \n",
              "                   2    0.088509  0.090446  0.024554  0.152464   \n",
              "4 tdvp             1    0.837968  0.199199  0.434681  0.955152   \n",
              "                   2    0.782392  0.116077  0.602306  0.896681   \n",
              "                   3    0.593010  0.182262  0.372322  0.862888   \n",
              "                   4    0.675338  0.099432  0.564874  0.806639   \n",
              "\n",
              "                     expectation approximation ratio per circuits            \\\n",
              "                                                             mean       std   \n",
              "n Algorithm        p                                                          \n",
              "3 tdvp             1                                     0.000309 -0.000002   \n",
              "                   2                                     0.000069 -0.000013   \n",
              "  scipy            1                                     0.031795  0.003681   \n",
              "                   2                                     0.018609  0.000647   \n",
              "  gradient_descent 1                                     0.001293  0.001826   \n",
              "                   2                                     0.000857  0.000063   \n",
              "4 tdvp             1                                     0.000068  0.000033   \n",
              "                   2                                     0.000012 -0.000012   \n",
              "                   3                                     0.000002 -0.000003   \n",
              "                   4                                     0.000006 -0.000001   \n",
              "\n",
              "                     relevance                  \n",
              "                       success total     ratio  \n",
              "n Algorithm        p                            \n",
              "3 tdvp             1         2     2       1.0  \n",
              "                   2         2     2       1.0  \n",
              "  scipy            1         2     2       1.0  \n",
              "                   2         2     2       1.0  \n",
              "  gradient_descent 1         2     2       1.0  \n",
              "                   2         2     2       1.0  \n",
              "4 tdvp             1         6     6       1.0  \n",
              "                   2         6     6       1.0  \n",
              "                   3         5     6  0.833333  \n",
              "                   4         4     6  0.666667  \n",
              "\n",
              "[10 rows x 41 columns]"
            ]
          },
          "execution_count": 154,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "expectation approximation ratio               mean    5.556973e-01\n",
              "                                              std     3.201487e-01\n",
              "                                              min     1.338465e-01\n",
              "                                              max     9.632252e-01\n",
              "groundspace overlap                           mean    4.643440e-01\n",
              "                                              std     3.731296e-01\n",
              "                                              min     1.987056e-02\n",
              "                                              max     9.623940e-01\n",
              "groundspace sharpness                         mean    3.318648e-01\n",
              "                                              std     1.645832e-01\n",
              "                                              min     1.395554e-01\n",
              "                                              max     4.977434e-01\n",
              "approximation ratio                           mean    8.500000e-01\n",
              "                                              std     2.760878e-01\n",
              "                                              min     3.142857e-01\n",
              "                                              max     1.000000e+00\n",
              "circuit count                                 mean    8.173000e+03\n",
              "                                              std     7.897058e+02\n",
              "                                              min     6.697000e+03\n",
              "                                              max     8.857000e+03\n",
              "gate count                                    mean    2.294739e+06\n",
              "                                              std     4.768168e+05\n",
              "                                              min     1.540540e+06\n",
              "                                              max     2.826550e+06\n",
              "num_fun_calls                                 mean    5.500000e+01\n",
              "                                              std     7.014271e+00\n",
              "                                              min     4.400000e+01\n",
              "                                              max     6.200000e+01\n",
              "steps                                         mean    7.566667e+01\n",
              "                                              std     7.312090e+00\n",
              "                                              min     6.200000e+01\n",
              "                                              max     8.200000e+01\n",
              "path length                                   mean    8.379681e-01\n",
              "                                              std     1.991995e-01\n",
              "                                              min     4.346813e-01\n",
              "                                              max     9.551523e-01\n",
              "expectation approximation ratio per circuits  mean    6.799184e-05\n",
              "                                              std     3.260188e-05\n",
              "Name: (4, tdvp, 1), dtype: float64"
            ]
          },
          "execution_count": 134,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats.loc[4,'tdvp',1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "def cohens_d(m1:float,m2:float,s1:float,s2:float,n1:int,n2:int)->float:\n",
        "    return (m1-m2)/np.sqrt((n1-1)*s1**2+(n2-1)*s2**2)\n",
        "\n",
        "def hedges_g(m1:float,m2:float,s1:float,s2:float,n1:int,n2:int)->float:\n",
        "    s_star = np.sqrt(\n",
        "        ((n1-1)*s1**2+(n2-1)*s2**2)/(n1+n2-2)\n",
        "    )\n",
        "    if s_star==0: \n",
        "        s_star=1e-10\n",
        "    a = n1+n2-2\n",
        "    J = gamma(a/2)/(np.sqrt(a/2)*gamma(a/2-0.5))\n",
        "    return J*(m1-m2)/s_star\n",
        "\n",
        "def effect_size_mapping(col,stats,index=hedges_g):\n",
        "    n,p = col.name\n",
        "    for meas, comp in col.index:\n",
        "        tdvp_stats = stats.loc[n,'tdvp',p]\n",
        "        comp_stats = stats.loc[n,comp,p]\n",
        "        col[meas,comp] = index(\n",
        "            m1= tdvp_stats[meas,\"mean\"], m2=comp_stats[meas,\"mean\"],\n",
        "            s1=tdvp_stats[meas,\"std\"], s2=comp_stats[meas,\"std\"],\n",
        "            n1=tdvp_stats['relevance',\"count\"], n2=comp_stats['relevance',\"count\"],\n",
        "        )\n",
        "        \n",
        "def effect_size_categorize(g:float) -> Literal['negligible', 'small', 'medium', 'large']:\n",
        "    if g<0.2:\n",
        "        return \"negligible\"\n",
        "    elif g<0.5:\n",
        "        return \"small\"\n",
        "    elif g<0.8:\n",
        "        return \"medium\"\n",
        "    else:\n",
        "        return \"large\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Index(['approximation ratio', 'circuit count',\n",
              "       'expectation approximation ratio',\n",
              "       'expectation approximation ratio per circuits', 'gate count',\n",
              "       'groundspace overlap', 'groundspace sharpness', 'num_fun_calls',\n",
              "       'path length', 'relevance', 'steps'],\n",
              "      dtype='object')"
            ]
          },
          "execution_count": 163,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats.drop(columns=['relevance'],level=0, inplace=False).columns.levels[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Index(['approximation ratio', 'circuit count',\n",
              "       'expectation approximation ratio',\n",
              "       'expectation approximation ratio per circuits', 'gate count',\n",
              "       'groundspace overlap', 'groundspace sharpness', 'num_fun_calls',\n",
              "       'path length'],\n",
              "      dtype='object')"
            ]
          },
          "execution_count": 167,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats.columns.levels[0][:-2]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "MultiIndex([(3, 1),\n",
              "            (3, 2),\n",
              "            (4, 1),\n",
              "            (4, 2),\n",
              "            (4, 3),\n",
              "            (4, 4),\n",
              "            (4, 5),\n",
              "            (5, 1),\n",
              "            (5, 2),\n",
              "            (5, 3),\n",
              "            (5, 4)],\n",
              "           names=['n', 'p'])"
            ]
          },
          "execution_count": 168,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats.xs(\"tdvp\", level=\"Algorithm\", drop_level=True).index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"2\" halign=\"left\">approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">circuit count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">expectation approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">expectation approximation ratio per circuits</th>\n",
              "      <th colspan=\"2\" halign=\"left\">gate count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">groundspace overlap</th>\n",
              "      <th colspan=\"2\" halign=\"left\">groundspace sharpness</th>\n",
              "      <th colspan=\"2\" halign=\"left\">num_fun_calls</th>\n",
              "      <th colspan=\"2\" halign=\"left\">path length</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">5</th>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    approximation ratio                  circuit count                   \\\n",
              "                  scipy gradient_descent         scipy gradient_descent   \n",
              "n p                                                                       \n",
              "3 1                 NaN              NaN           NaN              NaN   \n",
              "  2                 NaN              NaN           NaN              NaN   \n",
              "4 1                 NaN              NaN           NaN              NaN   \n",
              "  2                 NaN              NaN           NaN              NaN   \n",
              "  3                 NaN              NaN           NaN              NaN   \n",
              "  4                 NaN              NaN           NaN              NaN   \n",
              "  5                 NaN              NaN           NaN              NaN   \n",
              "5 1                 NaN              NaN           NaN              NaN   \n",
              "  2                 NaN              NaN           NaN              NaN   \n",
              "  3                 NaN              NaN           NaN              NaN   \n",
              "  4                 NaN              NaN           NaN              NaN   \n",
              "\n",
              "    expectation approximation ratio                   \\\n",
              "                              scipy gradient_descent   \n",
              "n p                                                    \n",
              "3 1                             NaN              NaN   \n",
              "  2                             NaN              NaN   \n",
              "4 1                             NaN              NaN   \n",
              "  2                             NaN              NaN   \n",
              "  3                             NaN              NaN   \n",
              "  4                             NaN              NaN   \n",
              "  5                             NaN              NaN   \n",
              "5 1                             NaN              NaN   \n",
              "  2                             NaN              NaN   \n",
              "  3                             NaN              NaN   \n",
              "  4                             NaN              NaN   \n",
              "\n",
              "    expectation approximation ratio per circuits                  gate count  \\\n",
              "                                           scipy gradient_descent      scipy   \n",
              "n p                                                                            \n",
              "3 1                                          NaN              NaN        NaN   \n",
              "  2                                          NaN              NaN        NaN   \n",
              "4 1                                          NaN              NaN        NaN   \n",
              "  2                                          NaN              NaN        NaN   \n",
              "  3                                          NaN              NaN        NaN   \n",
              "  4                                          NaN              NaN        NaN   \n",
              "  5                                          NaN              NaN        NaN   \n",
              "5 1                                          NaN              NaN        NaN   \n",
              "  2                                          NaN              NaN        NaN   \n",
              "  3                                          NaN              NaN        NaN   \n",
              "  4                                          NaN              NaN        NaN   \n",
              "\n",
              "                     groundspace overlap                   \\\n",
              "    gradient_descent               scipy gradient_descent   \n",
              "n p                                                         \n",
              "3 1              NaN                 NaN              NaN   \n",
              "  2              NaN                 NaN              NaN   \n",
              "4 1              NaN                 NaN              NaN   \n",
              "  2              NaN                 NaN              NaN   \n",
              "  3              NaN                 NaN              NaN   \n",
              "  4              NaN                 NaN              NaN   \n",
              "  5              NaN                 NaN              NaN   \n",
              "5 1              NaN                 NaN              NaN   \n",
              "  2              NaN                 NaN              NaN   \n",
              "  3              NaN                 NaN              NaN   \n",
              "  4              NaN                 NaN              NaN   \n",
              "\n",
              "    groundspace sharpness                  num_fun_calls                   \\\n",
              "                    scipy gradient_descent         scipy gradient_descent   \n",
              "n p                                                                         \n",
              "3 1                   NaN              NaN           NaN              NaN   \n",
              "  2                   NaN              NaN           NaN              NaN   \n",
              "4 1                   NaN              NaN           NaN              NaN   \n",
              "  2                   NaN              NaN           NaN              NaN   \n",
              "  3                   NaN              NaN           NaN              NaN   \n",
              "  4                   NaN              NaN           NaN              NaN   \n",
              "  5                   NaN              NaN           NaN              NaN   \n",
              "5 1                   NaN              NaN           NaN              NaN   \n",
              "  2                   NaN              NaN           NaN              NaN   \n",
              "  3                   NaN              NaN           NaN              NaN   \n",
              "  4                   NaN              NaN           NaN              NaN   \n",
              "\n",
              "    path length                   \n",
              "          scipy gradient_descent  \n",
              "n p                               \n",
              "3 1         NaN              NaN  \n",
              "  2         NaN              NaN  \n",
              "4 1         NaN              NaN  \n",
              "  2         NaN              NaN  \n",
              "  3         NaN              NaN  \n",
              "  4         NaN              NaN  \n",
              "  5         NaN              NaN  \n",
              "5 1         NaN              NaN  \n",
              "  2         NaN              NaN  \n",
              "  3         NaN              NaN  \n",
              "  4         NaN              NaN  "
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "effect_size = pd.DataFrame(\n",
        "    index=stats.xs(\"tdvp\", level=\"Algorithm\", drop_level=True).index,\n",
        "    columns=pd.MultiIndex.from_product(\n",
        "        [\n",
        "            stats.columns.levels[0][:-2],\n",
        "            [\"scipy\", \"gradient_descent\"],\n",
        "        ]\n",
        "    ),\n",
        ")\n",
        "effect_size\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {},
      "outputs": [],
      "source": [
        "effect_size.apply(effect_size_mapping, axis=1, args=(stats,));"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "957646427.023719"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "hedges_g(1.1,1,0,0,10,10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>Algorithm</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"6\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.942809</td>\n",
              "      <td>-0.333333</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"15\" valign=\"top\">4</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>0.850000</td>\n",
              "      <td>0.276088</td>\n",
              "      <td>0.314286</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>0.869048</td>\n",
              "      <td>0.232189</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.964286</td>\n",
              "      <td>0.087482</td>\n",
              "      <td>0.785714</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>0.100000</td>\n",
              "      <td>0.761041</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.067460</td>\n",
              "      <td>0.263595</td>\n",
              "      <td>-0.428571</td>\n",
              "      <td>0.357143</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.504762</td>\n",
              "      <td>0.919035</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.134921</td>\n",
              "      <td>0.844573</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.151587</td>\n",
              "      <td>0.755871</td>\n",
              "      <td>-0.928571</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"12\" valign=\"top\">5</th>\n",
              "      <th rowspan=\"4\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>0.515238</td>\n",
              "      <td>0.445836</td>\n",
              "      <td>-0.190476</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.923810</td>\n",
              "      <td>0.228571</td>\n",
              "      <td>0.314286</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>0.779592</td>\n",
              "      <td>0.260607</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.879592</td>\n",
              "      <td>0.200062</td>\n",
              "      <td>0.510204</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.928571</td>\n",
              "      <td>0.154303</td>\n",
              "      <td>0.571429</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.971429</td>\n",
              "      <td>0.060234</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>-0.231633</td>\n",
              "      <td>0.488527</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>0.314286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.234127</td>\n",
              "      <td>0.731186</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.158503</td>\n",
              "      <td>0.751500</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.171020</td>\n",
              "      <td>0.895167</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                          mean       std       min       max\n",
              "n Algorithm        p                                        \n",
              "3 tdvp             1  1.000000  0.000000  1.000000  1.000000\n",
              "                   2  1.000000  0.000000  1.000000  1.000000\n",
              "  scipy            1  1.000000  0.000000  1.000000  1.000000\n",
              "                   2  1.000000  0.000000  1.000000  1.000000\n",
              "  gradient_descent 1  0.333333  0.942809 -0.333333  1.000000\n",
              "                   2  1.000000  0.000000  1.000000  1.000000\n",
              "4 tdvp             1  0.850000  0.276088  0.314286  1.000000\n",
              "                   2  1.000000  0.000000  1.000000  1.000000\n",
              "                   3  1.000000  0.000000  1.000000  1.000000\n",
              "                   4  1.000000  0.000000  1.000000  1.000000\n",
              "                   5  1.000000  0.000000  1.000000  1.000000\n",
              "  scipy            1  0.869048  0.232189  0.428571  1.000000\n",
              "                   2  1.000000  0.000000  1.000000  1.000000\n",
              "                   3  1.000000  0.000000  1.000000  1.000000\n",
              "                   4  0.964286  0.087482  0.785714  1.000000\n",
              "                   5  1.000000  0.000000  1.000000  1.000000\n",
              "  gradient_descent 1  0.100000  0.761041 -1.285714  1.000000\n",
              "                   2  0.067460  0.263595 -0.428571  0.357143\n",
              "                   3  0.504762  0.919035 -1.285714  1.000000\n",
              "                   4  0.134921  0.844573 -1.285714  1.000000\n",
              "                   5  0.151587  0.755871 -0.928571  1.000000\n",
              "5 tdvp             1  0.515238  0.445836 -0.190476  1.000000\n",
              "                   2  0.923810  0.228571  0.314286  1.000000\n",
              "                   3  1.000000  0.000000  1.000000  1.000000\n",
              "                   4  1.000000  0.000000  1.000000  1.000000\n",
              "  scipy            1  0.779592  0.260607  0.428571  1.000000\n",
              "                   2  0.879592  0.200062  0.510204  1.000000\n",
              "                   3  0.928571  0.154303  0.571429  1.000000\n",
              "                   4  0.971429  0.060234  0.857143  1.000000\n",
              "  gradient_descent 1 -0.231633  0.488527 -1.285714  0.314286\n",
              "                   2  0.234127  0.731186 -1.285714  1.000000\n",
              "                   3 -0.158503  0.751500 -1.285714  1.000000\n",
              "                   4  0.171020  0.895167 -1.285714  1.000000"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats['approximation ratio']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [],
      "source": [
        "effect_size.drop(columns=['num_fun_calls'],level=0, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {},
      "outputs": [],
      "source": [
        "effect_size.rename(lambda x: x.replace(\"_\",\" \"), axis=1, level=0, inplace=True)\n",
        "effect_size.rename(lambda x: x.replace(\"gradient_descent\",\"GD\"), axis=1, level=1, inplace=True)\n",
        "effect_size.rename(lambda x: x.replace(\"scipy\",\"COBYLA\"), axis=1, level=1, inplace=True)\n",
        "effect_size.drop(index=[3], level=0, inplace=True)\n",
        "effect_size_directed = effect_size.copy()\n",
        "effect_size_undirected = effect_size.copy().apply(np.abs)\n",
        "effect_size_category = effect_size_undirected.copy().apply(lambda x: x.apply(effect_size_categorize))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"2\" halign=\"left\">Approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Circuit count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Expectation approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Expectation approximation ratio per circuits</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Gate count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Groundspace overlap</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Groundspace sharpness</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Path length</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>1</th>\n",
              "      <td>-0.068903</td>\n",
              "      <td>1.208933</td>\n",
              "      <td>13.466604</td>\n",
              "      <td>13.140384</td>\n",
              "      <td>-0.078062</td>\n",
              "      <td>1.790508</td>\n",
              "      <td>-5.955991</td>\n",
              "      <td>-0.908317</td>\n",
              "      <td>6.273362</td>\n",
              "      <td>6.019495</td>\n",
              "      <td>0.205933</td>\n",
              "      <td>1.18325</td>\n",
              "      <td>-0.080837</td>\n",
              "      <td>0.173569</td>\n",
              "      <td>4.196836</td>\n",
              "      <td>1.591686</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>4.616655</td>\n",
              "      <td>1.317935</td>\n",
              "      <td>1.292338</td>\n",
              "      <td>0.579214</td>\n",
              "      <td>6.59108</td>\n",
              "      <td>-4.797657</td>\n",
              "      <td>0.170779</td>\n",
              "      <td>1.097066</td>\n",
              "      <td>1.089854</td>\n",
              "      <td>0.596743</td>\n",
              "      <td>18.442253</td>\n",
              "      <td>-0.463243</td>\n",
              "      <td>1.317026</td>\n",
              "      <td>7.458731</td>\n",
              "      <td>7.132095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.660701</td>\n",
              "      <td>0.851146</td>\n",
              "      <td>0.840156</td>\n",
              "      <td>0.730455</td>\n",
              "      <td>3.025058</td>\n",
              "      <td>-2.671111</td>\n",
              "      <td>-0.03776</td>\n",
              "      <td>0.900838</td>\n",
              "      <td>0.898831</td>\n",
              "      <td>0.568261</td>\n",
              "      <td>5.375869</td>\n",
              "      <td>-0.396015</td>\n",
              "      <td>-0.174018</td>\n",
              "      <td>3.63329</td>\n",
              "      <td>4.053178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.466154</td>\n",
              "      <td>1.169564</td>\n",
              "      <td>5.057729</td>\n",
              "      <td>5.021144</td>\n",
              "      <td>0.857109</td>\n",
              "      <td>4.552017</td>\n",
              "      <td>-21.157234</td>\n",
              "      <td>0.338789</td>\n",
              "      <td>4.317769</td>\n",
              "      <td>4.315232</td>\n",
              "      <td>0.783773</td>\n",
              "      <td>4.534492</td>\n",
              "      <td>-0.569897</td>\n",
              "      <td>-0.375818</td>\n",
              "      <td>8.353581</td>\n",
              "      <td>7.575652</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.0</td>\n",
              "      <td>1.179602</td>\n",
              "      <td>1.709059</td>\n",
              "      <td>1.700069</td>\n",
              "      <td>0.37858</td>\n",
              "      <td>4.622825</td>\n",
              "      <td>-2.091172</td>\n",
              "      <td>0.360929</td>\n",
              "      <td>1.92263</td>\n",
              "      <td>1.922136</td>\n",
              "      <td>0.433454</td>\n",
              "      <td>6.400803</td>\n",
              "      <td>-0.352469</td>\n",
              "      <td>-0.599815</td>\n",
              "      <td>4.003324</td>\n",
              "      <td>4.15597</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">5</th>\n",
              "      <th>1</th>\n",
              "      <td>-0.693276</td>\n",
              "      <td>1.52937</td>\n",
              "      <td>4.100889</td>\n",
              "      <td>4.0591</td>\n",
              "      <td>-0.48583</td>\n",
              "      <td>1.535451</td>\n",
              "      <td>-5.453887</td>\n",
              "      <td>-1.01474</td>\n",
              "      <td>2.750824</td>\n",
              "      <td>2.665073</td>\n",
              "      <td>-0.156747</td>\n",
              "      <td>0.752066</td>\n",
              "      <td>-0.557466</td>\n",
              "      <td>0.588297</td>\n",
              "      <td>9.405778</td>\n",
              "      <td>2.13748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.197394</td>\n",
              "      <td>1.187657</td>\n",
              "      <td>2.056962</td>\n",
              "      <td>2.041357</td>\n",
              "      <td>1.01513</td>\n",
              "      <td>4.126366</td>\n",
              "      <td>-14.1204</td>\n",
              "      <td>-0.440118</td>\n",
              "      <td>1.615595</td>\n",
              "      <td>1.607239</td>\n",
              "      <td>1.091701</td>\n",
              "      <td>3.504858</td>\n",
              "      <td>0.011083</td>\n",
              "      <td>0.873125</td>\n",
              "      <td>3.944524</td>\n",
              "      <td>3.772738</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.567141</td>\n",
              "      <td>1.888697</td>\n",
              "      <td>4.461406</td>\n",
              "      <td>4.442341</td>\n",
              "      <td>0.872262</td>\n",
              "      <td>6.162096</td>\n",
              "      <td>-30.041762</td>\n",
              "      <td>0.301245</td>\n",
              "      <td>3.247997</td>\n",
              "      <td>3.243967</td>\n",
              "      <td>0.808675</td>\n",
              "      <td>6.080383</td>\n",
              "      <td>0.107252</td>\n",
              "      <td>1.351707</td>\n",
              "      <td>5.627975</td>\n",
              "      <td>5.455648</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.55924</td>\n",
              "      <td>1.091809</td>\n",
              "      <td>1.061756</td>\n",
              "      <td>1.058404</td>\n",
              "      <td>-0.011778</td>\n",
              "      <td>4.478561</td>\n",
              "      <td>-5.819317</td>\n",
              "      <td>0.022308</td>\n",
              "      <td>1.055591</td>\n",
              "      <td>1.05508</td>\n",
              "      <td>0.119516</td>\n",
              "      <td>3.857116</td>\n",
              "      <td>0.527886</td>\n",
              "      <td>1.531015</td>\n",
              "      <td>6.888549</td>\n",
              "      <td>7.053959</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Approximation ratio           Circuit count             \\\n",
              "                 COBYLA        GD        COBYLA         GD   \n",
              "n p                                                          \n",
              "4 1           -0.068903  1.208933     13.466604  13.140384   \n",
              "  2                 0.0  4.616655      1.317935   1.292338   \n",
              "  3                 0.0  0.660701      0.851146   0.840156   \n",
              "  4            0.466154  1.169564      5.057729   5.021144   \n",
              "  5                 0.0  1.179602      1.709059   1.700069   \n",
              "5 1           -0.693276   1.52937      4.100889     4.0591   \n",
              "  2            0.197394  1.187657      2.056962   2.041357   \n",
              "  3            0.567141  1.888697      4.461406   4.442341   \n",
              "  4             0.55924  1.091809      1.061756   1.058404   \n",
              "\n",
              "    Expectation approximation ratio            \\\n",
              "                             COBYLA        GD   \n",
              "n p                                             \n",
              "4 1                       -0.078062  1.790508   \n",
              "  2                        0.579214   6.59108   \n",
              "  3                        0.730455  3.025058   \n",
              "  4                        0.857109  4.552017   \n",
              "  5                         0.37858  4.622825   \n",
              "5 1                        -0.48583  1.535451   \n",
              "  2                         1.01513  4.126366   \n",
              "  3                        0.872262  6.162096   \n",
              "  4                       -0.011778  4.478561   \n",
              "\n",
              "    Expectation approximation ratio per circuits           Gate count  \\\n",
              "                                          COBYLA        GD     COBYLA   \n",
              "n p                                                                     \n",
              "4 1                                    -5.955991 -0.908317   6.273362   \n",
              "  2                                    -4.797657  0.170779   1.097066   \n",
              "  3                                    -2.671111  -0.03776   0.900838   \n",
              "  4                                   -21.157234  0.338789   4.317769   \n",
              "  5                                    -2.091172  0.360929    1.92263   \n",
              "5 1                                    -5.453887  -1.01474   2.750824   \n",
              "  2                                     -14.1204 -0.440118   1.615595   \n",
              "  3                                   -30.041762  0.301245   3.247997   \n",
              "  4                                    -5.819317  0.022308   1.055591   \n",
              "\n",
              "              Groundspace overlap            Groundspace sharpness            \\\n",
              "           GD              COBYLA         GD                COBYLA        GD   \n",
              "n p                                                                            \n",
              "4 1  6.019495            0.205933    1.18325             -0.080837  0.173569   \n",
              "  2  1.089854            0.596743  18.442253             -0.463243  1.317026   \n",
              "  3  0.898831            0.568261   5.375869             -0.396015 -0.174018   \n",
              "  4  4.315232            0.783773   4.534492             -0.569897 -0.375818   \n",
              "  5  1.922136            0.433454   6.400803             -0.352469 -0.599815   \n",
              "5 1  2.665073           -0.156747   0.752066             -0.557466  0.588297   \n",
              "  2  1.607239            1.091701   3.504858              0.011083  0.873125   \n",
              "  3  3.243967            0.808675   6.080383              0.107252  1.351707   \n",
              "  4   1.05508            0.119516   3.857116              0.527886  1.531015   \n",
              "\n",
              "    Path length            \n",
              "         COBYLA        GD  \n",
              "n p                        \n",
              "4 1    4.196836  1.591686  \n",
              "  2    7.458731  7.132095  \n",
              "  3     3.63329  4.053178  \n",
              "  4    8.353581  7.575652  \n",
              "  5    4.003324   4.15597  \n",
              "5 1    9.405778   2.13748  \n",
              "  2    3.944524  3.772738  \n",
              "  3    5.627975  5.455648  \n",
              "  4    6.888549  7.053959  "
            ]
          },
          "execution_count": 59,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "effect_size_directed"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 90,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"2\" halign=\"left\">Approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Circuit count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Expectation approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Expectation approximation ratio per circuits</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Gate count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Groundspace overlap</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Groundspace sharpness</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Path length</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>1</th>\n",
              "      <td>0.068903</td>\n",
              "      <td>1.208933</td>\n",
              "      <td>13.466604</td>\n",
              "      <td>13.140384</td>\n",
              "      <td>0.078062</td>\n",
              "      <td>1.790508</td>\n",
              "      <td>5.955991</td>\n",
              "      <td>0.908317</td>\n",
              "      <td>6.273362</td>\n",
              "      <td>6.019495</td>\n",
              "      <td>0.205933</td>\n",
              "      <td>1.18325</td>\n",
              "      <td>0.080837</td>\n",
              "      <td>0.173569</td>\n",
              "      <td>4.196836</td>\n",
              "      <td>1.591686</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>4.616655</td>\n",
              "      <td>1.317935</td>\n",
              "      <td>1.292338</td>\n",
              "      <td>0.579214</td>\n",
              "      <td>6.59108</td>\n",
              "      <td>4.797657</td>\n",
              "      <td>0.170779</td>\n",
              "      <td>1.097066</td>\n",
              "      <td>1.089854</td>\n",
              "      <td>0.596743</td>\n",
              "      <td>18.442253</td>\n",
              "      <td>0.463243</td>\n",
              "      <td>1.317026</td>\n",
              "      <td>7.458731</td>\n",
              "      <td>7.132095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.660701</td>\n",
              "      <td>0.851146</td>\n",
              "      <td>0.840156</td>\n",
              "      <td>0.730455</td>\n",
              "      <td>3.025058</td>\n",
              "      <td>2.671111</td>\n",
              "      <td>0.03776</td>\n",
              "      <td>0.900838</td>\n",
              "      <td>0.898831</td>\n",
              "      <td>0.568261</td>\n",
              "      <td>5.375869</td>\n",
              "      <td>0.396015</td>\n",
              "      <td>0.174018</td>\n",
              "      <td>3.63329</td>\n",
              "      <td>4.053178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.466154</td>\n",
              "      <td>1.169564</td>\n",
              "      <td>5.057729</td>\n",
              "      <td>5.021144</td>\n",
              "      <td>0.857109</td>\n",
              "      <td>4.552017</td>\n",
              "      <td>21.157234</td>\n",
              "      <td>0.338789</td>\n",
              "      <td>4.317769</td>\n",
              "      <td>4.315232</td>\n",
              "      <td>0.783773</td>\n",
              "      <td>4.534492</td>\n",
              "      <td>0.569897</td>\n",
              "      <td>0.375818</td>\n",
              "      <td>8.353581</td>\n",
              "      <td>7.575652</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.0</td>\n",
              "      <td>1.179602</td>\n",
              "      <td>1.709059</td>\n",
              "      <td>1.700069</td>\n",
              "      <td>0.37858</td>\n",
              "      <td>4.622825</td>\n",
              "      <td>2.091172</td>\n",
              "      <td>0.360929</td>\n",
              "      <td>1.92263</td>\n",
              "      <td>1.922136</td>\n",
              "      <td>0.433454</td>\n",
              "      <td>6.400803</td>\n",
              "      <td>0.352469</td>\n",
              "      <td>0.599815</td>\n",
              "      <td>4.003324</td>\n",
              "      <td>4.15597</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">5</th>\n",
              "      <th>1</th>\n",
              "      <td>0.693276</td>\n",
              "      <td>1.52937</td>\n",
              "      <td>4.100889</td>\n",
              "      <td>4.0591</td>\n",
              "      <td>0.48583</td>\n",
              "      <td>1.535451</td>\n",
              "      <td>5.453887</td>\n",
              "      <td>1.01474</td>\n",
              "      <td>2.750824</td>\n",
              "      <td>2.665073</td>\n",
              "      <td>0.156747</td>\n",
              "      <td>0.752066</td>\n",
              "      <td>0.557466</td>\n",
              "      <td>0.588297</td>\n",
              "      <td>9.405778</td>\n",
              "      <td>2.13748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.197394</td>\n",
              "      <td>1.187657</td>\n",
              "      <td>2.056962</td>\n",
              "      <td>2.041357</td>\n",
              "      <td>1.01513</td>\n",
              "      <td>4.126366</td>\n",
              "      <td>14.1204</td>\n",
              "      <td>0.440118</td>\n",
              "      <td>1.615595</td>\n",
              "      <td>1.607239</td>\n",
              "      <td>1.091701</td>\n",
              "      <td>3.504858</td>\n",
              "      <td>0.011083</td>\n",
              "      <td>0.873125</td>\n",
              "      <td>3.944524</td>\n",
              "      <td>3.772738</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.567141</td>\n",
              "      <td>1.888697</td>\n",
              "      <td>4.461406</td>\n",
              "      <td>4.442341</td>\n",
              "      <td>0.872262</td>\n",
              "      <td>6.162096</td>\n",
              "      <td>30.041762</td>\n",
              "      <td>0.301245</td>\n",
              "      <td>3.247997</td>\n",
              "      <td>3.243967</td>\n",
              "      <td>0.808675</td>\n",
              "      <td>6.080383</td>\n",
              "      <td>0.107252</td>\n",
              "      <td>1.351707</td>\n",
              "      <td>5.627975</td>\n",
              "      <td>5.455648</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.55924</td>\n",
              "      <td>1.091809</td>\n",
              "      <td>1.061756</td>\n",
              "      <td>1.058404</td>\n",
              "      <td>0.011778</td>\n",
              "      <td>4.478561</td>\n",
              "      <td>5.819317</td>\n",
              "      <td>0.022308</td>\n",
              "      <td>1.055591</td>\n",
              "      <td>1.05508</td>\n",
              "      <td>0.119516</td>\n",
              "      <td>3.857116</td>\n",
              "      <td>0.527886</td>\n",
              "      <td>1.531015</td>\n",
              "      <td>6.888549</td>\n",
              "      <td>7.053959</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Approximation ratio           Circuit count             \\\n",
              "                 COBYLA        GD        COBYLA         GD   \n",
              "n p                                                          \n",
              "4 1            0.068903  1.208933     13.466604  13.140384   \n",
              "  2                 0.0  4.616655      1.317935   1.292338   \n",
              "  3                 0.0  0.660701      0.851146   0.840156   \n",
              "  4            0.466154  1.169564      5.057729   5.021144   \n",
              "  5                 0.0  1.179602      1.709059   1.700069   \n",
              "5 1            0.693276   1.52937      4.100889     4.0591   \n",
              "  2            0.197394  1.187657      2.056962   2.041357   \n",
              "  3            0.567141  1.888697      4.461406   4.442341   \n",
              "  4             0.55924  1.091809      1.061756   1.058404   \n",
              "\n",
              "    Expectation approximation ratio            \\\n",
              "                             COBYLA        GD   \n",
              "n p                                             \n",
              "4 1                        0.078062  1.790508   \n",
              "  2                        0.579214   6.59108   \n",
              "  3                        0.730455  3.025058   \n",
              "  4                        0.857109  4.552017   \n",
              "  5                         0.37858  4.622825   \n",
              "5 1                         0.48583  1.535451   \n",
              "  2                         1.01513  4.126366   \n",
              "  3                        0.872262  6.162096   \n",
              "  4                        0.011778  4.478561   \n",
              "\n",
              "    Expectation approximation ratio per circuits           Gate count  \\\n",
              "                                          COBYLA        GD     COBYLA   \n",
              "n p                                                                     \n",
              "4 1                                     5.955991  0.908317   6.273362   \n",
              "  2                                     4.797657  0.170779   1.097066   \n",
              "  3                                     2.671111   0.03776   0.900838   \n",
              "  4                                    21.157234  0.338789   4.317769   \n",
              "  5                                     2.091172  0.360929    1.92263   \n",
              "5 1                                     5.453887   1.01474   2.750824   \n",
              "  2                                      14.1204  0.440118   1.615595   \n",
              "  3                                    30.041762  0.301245   3.247997   \n",
              "  4                                     5.819317  0.022308   1.055591   \n",
              "\n",
              "              Groundspace overlap            Groundspace sharpness            \\\n",
              "           GD              COBYLA         GD                COBYLA        GD   \n",
              "n p                                                                            \n",
              "4 1  6.019495            0.205933    1.18325              0.080837  0.173569   \n",
              "  2  1.089854            0.596743  18.442253              0.463243  1.317026   \n",
              "  3  0.898831            0.568261   5.375869              0.396015  0.174018   \n",
              "  4  4.315232            0.783773   4.534492              0.569897  0.375818   \n",
              "  5  1.922136            0.433454   6.400803              0.352469  0.599815   \n",
              "5 1  2.665073            0.156747   0.752066              0.557466  0.588297   \n",
              "  2  1.607239            1.091701   3.504858              0.011083  0.873125   \n",
              "  3  3.243967            0.808675   6.080383              0.107252  1.351707   \n",
              "  4   1.05508            0.119516   3.857116              0.527886  1.531015   \n",
              "\n",
              "    Path length            \n",
              "         COBYLA        GD  \n",
              "n p                        \n",
              "4 1    4.196836  1.591686  \n",
              "  2    7.458731  7.132095  \n",
              "  3     3.63329  4.053178  \n",
              "  4    8.353581  7.575652  \n",
              "  5    4.003324   4.15597  \n",
              "5 1    9.405778   2.13748  \n",
              "  2    3.944524  3.772738  \n",
              "  3    5.627975  5.455648  \n",
              "  4    6.888549  7.053959  "
            ]
          },
          "execution_count": 90,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "effect_size_undirected"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"2\" halign=\"left\">Approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Circuit count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Expectation approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Expectation approximation ratio per circuits</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Gate count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Groundspace overlap</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Groundspace sharpness</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Path length</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>1</th>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>negligible</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>medium</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">5</th>\n",
              "      <th>1</th>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>medium</td>\n",
              "      <td>medium</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>small</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>negligible</td>\n",
              "      <td>large</td>\n",
              "      <td>medium</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "      <td>large</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Approximation ratio         Circuit count         \\\n",
              "                 COBYLA      GD        COBYLA     GD   \n",
              "n p                                                    \n",
              "4 1          negligible   large         large  large   \n",
              "  2          negligible   large         large  large   \n",
              "  3          negligible  medium         large  large   \n",
              "  4               small   large         large  large   \n",
              "  5          negligible   large         large  large   \n",
              "5 1              medium   large         large  large   \n",
              "  2          negligible   large         large  large   \n",
              "  3              medium   large         large  large   \n",
              "  4              medium   large         large  large   \n",
              "\n",
              "    Expectation approximation ratio         \\\n",
              "                             COBYLA     GD   \n",
              "n p                                          \n",
              "4 1                      negligible  large   \n",
              "  2                          medium  large   \n",
              "  3                          medium  large   \n",
              "  4                           large  large   \n",
              "  5                           small  large   \n",
              "5 1                           small  large   \n",
              "  2                           large  large   \n",
              "  3                           large  large   \n",
              "  4                      negligible  large   \n",
              "\n",
              "    Expectation approximation ratio per circuits             Gate count  \\\n",
              "                                          COBYLA          GD     COBYLA   \n",
              "n p                                                                       \n",
              "4 1                                        large       large      large   \n",
              "  2                                        large  negligible      large   \n",
              "  3                                        large  negligible      large   \n",
              "  4                                        large       small      large   \n",
              "  5                                        large       small      large   \n",
              "5 1                                        large       large      large   \n",
              "  2                                        large       small      large   \n",
              "  3                                        large       small      large   \n",
              "  4                                        large  negligible      large   \n",
              "\n",
              "           Groundspace overlap         Groundspace sharpness              \\\n",
              "        GD              COBYLA      GD                COBYLA          GD   \n",
              "n p                                                                        \n",
              "4 1  large               small   large            negligible  negligible   \n",
              "  2  large              medium   large                 small       large   \n",
              "  3  large              medium   large                 small  negligible   \n",
              "  4  large              medium   large                medium       small   \n",
              "  5  large               small   large                 small      medium   \n",
              "5 1  large          negligible  medium                medium      medium   \n",
              "  2  large               large   large            negligible       large   \n",
              "  3  large               large   large            negligible       large   \n",
              "  4  large          negligible   large                medium       large   \n",
              "\n",
              "    Path length         \n",
              "         COBYLA     GD  \n",
              "n p                     \n",
              "4 1       large  large  \n",
              "  2       large  large  \n",
              "  3       large  large  \n",
              "  4       large  large  \n",
              "  5       large  large  \n",
              "5 1       large  large  \n",
              "  2       large  large  \n",
              "  3       large  large  \n",
              "  4       large  large  "
            ]
          },
          "execution_count": 74,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "effect_size_category"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 89,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'-1.23'"
            ]
          },
          "execution_count": 89,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\"{:.2f}\".format(-1.2342584765874692765982769827568914356713784650134876)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 91,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"2\" halign=\"left\">Approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Circuit count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Expectation approximation ratio</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Expectation approximation ratio per circuits</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Gate count</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Groundspace overlap</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Groundspace sharpness</th>\n",
              "      <th colspan=\"2\" halign=\"left\">Path length</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>1</th>\n",
              "      <td>negligible (-0.07)</td>\n",
              "      <td>large (1.21)</td>\n",
              "      <td>large (13.47)</td>\n",
              "      <td>large (13.14)</td>\n",
              "      <td>negligible (-0.08)</td>\n",
              "      <td>large (1.79)</td>\n",
              "      <td>large (-5.96)</td>\n",
              "      <td>large (-0.91)</td>\n",
              "      <td>large (6.27)</td>\n",
              "      <td>large (6.02)</td>\n",
              "      <td>small (0.21)</td>\n",
              "      <td>large (1.18)</td>\n",
              "      <td>negligible (-0.08)</td>\n",
              "      <td>negligible (0.17)</td>\n",
              "      <td>large (4.20)</td>\n",
              "      <td>large (1.59)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negligible (0.00)</td>\n",
              "      <td>large (4.62)</td>\n",
              "      <td>large (1.32)</td>\n",
              "      <td>large (1.29)</td>\n",
              "      <td>medium (0.58)</td>\n",
              "      <td>large (6.59)</td>\n",
              "      <td>large (-4.80)</td>\n",
              "      <td>negligible (0.17)</td>\n",
              "      <td>large (1.10)</td>\n",
              "      <td>large (1.09)</td>\n",
              "      <td>medium (0.60)</td>\n",
              "      <td>large (18.44)</td>\n",
              "      <td>small (-0.46)</td>\n",
              "      <td>large (1.32)</td>\n",
              "      <td>large (7.46)</td>\n",
              "      <td>large (7.13)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>negligible (0.00)</td>\n",
              "      <td>medium (0.66)</td>\n",
              "      <td>large (0.85)</td>\n",
              "      <td>large (0.84)</td>\n",
              "      <td>medium (0.73)</td>\n",
              "      <td>large (3.03)</td>\n",
              "      <td>large (-2.67)</td>\n",
              "      <td>negligible (-0.04)</td>\n",
              "      <td>large (0.90)</td>\n",
              "      <td>large (0.90)</td>\n",
              "      <td>medium (0.57)</td>\n",
              "      <td>large (5.38)</td>\n",
              "      <td>small (-0.40)</td>\n",
              "      <td>negligible (-0.17)</td>\n",
              "      <td>large (3.63)</td>\n",
              "      <td>large (4.05)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>small (0.47)</td>\n",
              "      <td>large (1.17)</td>\n",
              "      <td>large (5.06)</td>\n",
              "      <td>large (5.02)</td>\n",
              "      <td>large (0.86)</td>\n",
              "      <td>large (4.55)</td>\n",
              "      <td>large (-21.16)</td>\n",
              "      <td>small (0.34)</td>\n",
              "      <td>large (4.32)</td>\n",
              "      <td>large (4.32)</td>\n",
              "      <td>medium (0.78)</td>\n",
              "      <td>large (4.53)</td>\n",
              "      <td>medium (-0.57)</td>\n",
              "      <td>small (-0.38)</td>\n",
              "      <td>large (8.35)</td>\n",
              "      <td>large (7.58)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>negligible (0.00)</td>\n",
              "      <td>large (1.18)</td>\n",
              "      <td>large (1.71)</td>\n",
              "      <td>large (1.70)</td>\n",
              "      <td>small (0.38)</td>\n",
              "      <td>large (4.62)</td>\n",
              "      <td>large (-2.09)</td>\n",
              "      <td>small (0.36)</td>\n",
              "      <td>large (1.92)</td>\n",
              "      <td>large (1.92)</td>\n",
              "      <td>small (0.43)</td>\n",
              "      <td>large (6.40)</td>\n",
              "      <td>small (-0.35)</td>\n",
              "      <td>medium (-0.60)</td>\n",
              "      <td>large (4.00)</td>\n",
              "      <td>large (4.16)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">5</th>\n",
              "      <th>1</th>\n",
              "      <td>medium (-0.69)</td>\n",
              "      <td>large (1.53)</td>\n",
              "      <td>large (4.10)</td>\n",
              "      <td>large (4.06)</td>\n",
              "      <td>small (-0.49)</td>\n",
              "      <td>large (1.54)</td>\n",
              "      <td>large (-5.45)</td>\n",
              "      <td>large (-1.01)</td>\n",
              "      <td>large (2.75)</td>\n",
              "      <td>large (2.67)</td>\n",
              "      <td>negligible (-0.16)</td>\n",
              "      <td>medium (0.75)</td>\n",
              "      <td>medium (-0.56)</td>\n",
              "      <td>medium (0.59)</td>\n",
              "      <td>large (9.41)</td>\n",
              "      <td>large (2.14)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negligible (0.20)</td>\n",
              "      <td>large (1.19)</td>\n",
              "      <td>large (2.06)</td>\n",
              "      <td>large (2.04)</td>\n",
              "      <td>large (1.02)</td>\n",
              "      <td>large (4.13)</td>\n",
              "      <td>large (-14.12)</td>\n",
              "      <td>small (-0.44)</td>\n",
              "      <td>large (1.62)</td>\n",
              "      <td>large (1.61)</td>\n",
              "      <td>large (1.09)</td>\n",
              "      <td>large (3.50)</td>\n",
              "      <td>negligible (0.01)</td>\n",
              "      <td>large (0.87)</td>\n",
              "      <td>large (3.94)</td>\n",
              "      <td>large (3.77)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>medium (0.57)</td>\n",
              "      <td>large (1.89)</td>\n",
              "      <td>large (4.46)</td>\n",
              "      <td>large (4.44)</td>\n",
              "      <td>large (0.87)</td>\n",
              "      <td>large (6.16)</td>\n",
              "      <td>large (-30.04)</td>\n",
              "      <td>small (0.30)</td>\n",
              "      <td>large (3.25)</td>\n",
              "      <td>large (3.24)</td>\n",
              "      <td>large (0.81)</td>\n",
              "      <td>large (6.08)</td>\n",
              "      <td>negligible (0.11)</td>\n",
              "      <td>large (1.35)</td>\n",
              "      <td>large (5.63)</td>\n",
              "      <td>large (5.46)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>medium (0.56)</td>\n",
              "      <td>large (1.09)</td>\n",
              "      <td>large (1.06)</td>\n",
              "      <td>large (1.06)</td>\n",
              "      <td>negligible (-0.01)</td>\n",
              "      <td>large (4.48)</td>\n",
              "      <td>large (-5.82)</td>\n",
              "      <td>negligible (0.02)</td>\n",
              "      <td>large (1.06)</td>\n",
              "      <td>large (1.06)</td>\n",
              "      <td>negligible (0.12)</td>\n",
              "      <td>large (3.86)</td>\n",
              "      <td>medium (0.53)</td>\n",
              "      <td>large (1.53)</td>\n",
              "      <td>large (6.89)</td>\n",
              "      <td>large (7.05)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Approximation ratio                 Circuit count                 \\\n",
              "                 COBYLA             GD         COBYLA             GD   \n",
              "n p                                                                    \n",
              "4 1  negligible (-0.07)   large (1.21)  large (13.47)  large (13.14)   \n",
              "  2   negligible (0.00)   large (4.62)   large (1.32)   large (1.29)   \n",
              "  3   negligible (0.00)  medium (0.66)   large (0.85)   large (0.84)   \n",
              "  4        small (0.47)   large (1.17)   large (5.06)   large (5.02)   \n",
              "  5   negligible (0.00)   large (1.18)   large (1.71)   large (1.70)   \n",
              "5 1      medium (-0.69)   large (1.53)   large (4.10)   large (4.06)   \n",
              "  2   negligible (0.20)   large (1.19)   large (2.06)   large (2.04)   \n",
              "  3       medium (0.57)   large (1.89)   large (4.46)   large (4.44)   \n",
              "  4       medium (0.56)   large (1.09)   large (1.06)   large (1.06)   \n",
              "\n",
              "    Expectation approximation ratio                \\\n",
              "                             COBYLA            GD   \n",
              "n p                                                 \n",
              "4 1              negligible (-0.08)  large (1.79)   \n",
              "  2                   medium (0.58)  large (6.59)   \n",
              "  3                   medium (0.73)  large (3.03)   \n",
              "  4                    large (0.86)  large (4.55)   \n",
              "  5                    small (0.38)  large (4.62)   \n",
              "5 1                   small (-0.49)  large (1.54)   \n",
              "  2                    large (1.02)  large (4.13)   \n",
              "  3                    large (0.87)  large (6.16)   \n",
              "  4              negligible (-0.01)  large (4.48)   \n",
              "\n",
              "    Expectation approximation ratio per circuits                      \\\n",
              "                                          COBYLA                  GD   \n",
              "n p                                                                    \n",
              "4 1                                large (-5.96)       large (-0.91)   \n",
              "  2                                large (-4.80)   negligible (0.17)   \n",
              "  3                                large (-2.67)  negligible (-0.04)   \n",
              "  4                               large (-21.16)        small (0.34)   \n",
              "  5                                large (-2.09)        small (0.36)   \n",
              "5 1                                large (-5.45)       large (-1.01)   \n",
              "  2                               large (-14.12)       small (-0.44)   \n",
              "  3                               large (-30.04)        small (0.30)   \n",
              "  4                                large (-5.82)   negligible (0.02)   \n",
              "\n",
              "       Gate count               Groundspace overlap                 \\\n",
              "           COBYLA            GD              COBYLA             GD   \n",
              "n p                                                                  \n",
              "4 1  large (6.27)  large (6.02)        small (0.21)   large (1.18)   \n",
              "  2  large (1.10)  large (1.09)       medium (0.60)  large (18.44)   \n",
              "  3  large (0.90)  large (0.90)       medium (0.57)   large (5.38)   \n",
              "  4  large (4.32)  large (4.32)       medium (0.78)   large (4.53)   \n",
              "  5  large (1.92)  large (1.92)        small (0.43)   large (6.40)   \n",
              "5 1  large (2.75)  large (2.67)  negligible (-0.16)  medium (0.75)   \n",
              "  2  large (1.62)  large (1.61)        large (1.09)   large (3.50)   \n",
              "  3  large (3.25)  large (3.24)        large (0.81)   large (6.08)   \n",
              "  4  large (1.06)  large (1.06)   negligible (0.12)   large (3.86)   \n",
              "\n",
              "    Groundspace sharpness                       Path length                \n",
              "                   COBYLA                  GD        COBYLA            GD  \n",
              "n p                                                                        \n",
              "4 1    negligible (-0.08)   negligible (0.17)  large (4.20)  large (1.59)  \n",
              "  2         small (-0.46)        large (1.32)  large (7.46)  large (7.13)  \n",
              "  3         small (-0.40)  negligible (-0.17)  large (3.63)  large (4.05)  \n",
              "  4        medium (-0.57)       small (-0.38)  large (8.35)  large (7.58)  \n",
              "  5         small (-0.35)      medium (-0.60)  large (4.00)  large (4.16)  \n",
              "5 1        medium (-0.56)       medium (0.59)  large (9.41)  large (2.14)  \n",
              "  2     negligible (0.01)        large (0.87)  large (3.94)  large (3.77)  \n",
              "  3     negligible (0.11)        large (1.35)  large (5.63)  large (5.46)  \n",
              "  4         medium (0.53)        large (1.53)  large (6.89)  large (7.05)  "
            ]
          },
          "execution_count": 91,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "effect_size_combined = effect_size_category.copy()\n",
        "for idx in effect_size_combined.index:\n",
        "    for meas, comp in effect_size_combined.loc[idx].index:\n",
        "        effect_size_combined.loc[idx][meas,comp] = effect_size_category.loc[idx][meas,comp] + \" (\" + \"{:.2f}\".format(effect_size_directed.loc[idx][meas,comp])+\")\"\n",
        "effect_size_combined"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 95,
      "metadata": {},
      "outputs": [],
      "source": [
        "meas_abbrv = {\n",
        "    'Approximation ratio': 'M1',\n",
        "    'Groundspace overlap': 'M2',\n",
        "    'Groundspace sharpness': 'M3',\n",
        "    'Expectation approximation ratio': 'M4',\n",
        "    'Gate count': 'M5',\n",
        "    'Circuit count': 'M6',\n",
        "    'Expectation approximation ratio per circuits': 'M7',\n",
        "    'Path length': 'M8',\n",
        "              }\n",
        "effect_size_combined.rename(meas_abbrv, axis=1, level=0, inplace=True)\n",
        "effect_size_combined.sort_index(axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 96,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"2\" halign=\"left\">M1</th>\n",
              "      <th colspan=\"2\" halign=\"left\">M2</th>\n",
              "      <th colspan=\"2\" halign=\"left\">M3</th>\n",
              "      <th colspan=\"2\" halign=\"left\">M4</th>\n",
              "      <th colspan=\"2\" halign=\"left\">M5</th>\n",
              "      <th colspan=\"2\" halign=\"left\">M6</th>\n",
              "      <th colspan=\"2\" halign=\"left\">M7</th>\n",
              "      <th colspan=\"2\" halign=\"left\">M8</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "      <th>COBYLA</th>\n",
              "      <th>GD</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>1</th>\n",
              "      <td>negligible (-0.07)</td>\n",
              "      <td>large (1.21)</td>\n",
              "      <td>small (0.21)</td>\n",
              "      <td>large (1.18)</td>\n",
              "      <td>negligible (-0.08)</td>\n",
              "      <td>negligible (0.17)</td>\n",
              "      <td>negligible (-0.08)</td>\n",
              "      <td>large (1.79)</td>\n",
              "      <td>large (6.27)</td>\n",
              "      <td>large (6.02)</td>\n",
              "      <td>large (13.47)</td>\n",
              "      <td>large (13.14)</td>\n",
              "      <td>large (-5.96)</td>\n",
              "      <td>large (-0.91)</td>\n",
              "      <td>large (4.20)</td>\n",
              "      <td>large (1.59)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negligible (0.00)</td>\n",
              "      <td>large (4.62)</td>\n",
              "      <td>medium (0.60)</td>\n",
              "      <td>large (18.44)</td>\n",
              "      <td>small (-0.46)</td>\n",
              "      <td>large (1.32)</td>\n",
              "      <td>medium (0.58)</td>\n",
              "      <td>large (6.59)</td>\n",
              "      <td>large (1.10)</td>\n",
              "      <td>large (1.09)</td>\n",
              "      <td>large (1.32)</td>\n",
              "      <td>large (1.29)</td>\n",
              "      <td>large (-4.80)</td>\n",
              "      <td>negligible (0.17)</td>\n",
              "      <td>large (7.46)</td>\n",
              "      <td>large (7.13)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>negligible (0.00)</td>\n",
              "      <td>medium (0.66)</td>\n",
              "      <td>medium (0.57)</td>\n",
              "      <td>large (5.38)</td>\n",
              "      <td>small (-0.40)</td>\n",
              "      <td>negligible (-0.17)</td>\n",
              "      <td>medium (0.73)</td>\n",
              "      <td>large (3.03)</td>\n",
              "      <td>large (0.90)</td>\n",
              "      <td>large (0.90)</td>\n",
              "      <td>large (0.85)</td>\n",
              "      <td>large (0.84)</td>\n",
              "      <td>large (-2.67)</td>\n",
              "      <td>negligible (-0.04)</td>\n",
              "      <td>large (3.63)</td>\n",
              "      <td>large (4.05)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>small (0.47)</td>\n",
              "      <td>large (1.17)</td>\n",
              "      <td>medium (0.78)</td>\n",
              "      <td>large (4.53)</td>\n",
              "      <td>medium (-0.57)</td>\n",
              "      <td>small (-0.38)</td>\n",
              "      <td>large (0.86)</td>\n",
              "      <td>large (4.55)</td>\n",
              "      <td>large (4.32)</td>\n",
              "      <td>large (4.32)</td>\n",
              "      <td>large (5.06)</td>\n",
              "      <td>large (5.02)</td>\n",
              "      <td>large (-21.16)</td>\n",
              "      <td>small (0.34)</td>\n",
              "      <td>large (8.35)</td>\n",
              "      <td>large (7.58)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>negligible (0.00)</td>\n",
              "      <td>large (1.18)</td>\n",
              "      <td>small (0.43)</td>\n",
              "      <td>large (6.40)</td>\n",
              "      <td>small (-0.35)</td>\n",
              "      <td>medium (-0.60)</td>\n",
              "      <td>small (0.38)</td>\n",
              "      <td>large (4.62)</td>\n",
              "      <td>large (1.92)</td>\n",
              "      <td>large (1.92)</td>\n",
              "      <td>large (1.71)</td>\n",
              "      <td>large (1.70)</td>\n",
              "      <td>large (-2.09)</td>\n",
              "      <td>small (0.36)</td>\n",
              "      <td>large (4.00)</td>\n",
              "      <td>large (4.16)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">5</th>\n",
              "      <th>1</th>\n",
              "      <td>medium (-0.69)</td>\n",
              "      <td>large (1.53)</td>\n",
              "      <td>negligible (-0.16)</td>\n",
              "      <td>medium (0.75)</td>\n",
              "      <td>medium (-0.56)</td>\n",
              "      <td>medium (0.59)</td>\n",
              "      <td>small (-0.49)</td>\n",
              "      <td>large (1.54)</td>\n",
              "      <td>large (2.75)</td>\n",
              "      <td>large (2.67)</td>\n",
              "      <td>large (4.10)</td>\n",
              "      <td>large (4.06)</td>\n",
              "      <td>large (-5.45)</td>\n",
              "      <td>large (-1.01)</td>\n",
              "      <td>large (9.41)</td>\n",
              "      <td>large (2.14)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negligible (0.20)</td>\n",
              "      <td>large (1.19)</td>\n",
              "      <td>large (1.09)</td>\n",
              "      <td>large (3.50)</td>\n",
              "      <td>negligible (0.01)</td>\n",
              "      <td>large (0.87)</td>\n",
              "      <td>large (1.02)</td>\n",
              "      <td>large (4.13)</td>\n",
              "      <td>large (1.62)</td>\n",
              "      <td>large (1.61)</td>\n",
              "      <td>large (2.06)</td>\n",
              "      <td>large (2.04)</td>\n",
              "      <td>large (-14.12)</td>\n",
              "      <td>small (-0.44)</td>\n",
              "      <td>large (3.94)</td>\n",
              "      <td>large (3.77)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>medium (0.57)</td>\n",
              "      <td>large (1.89)</td>\n",
              "      <td>large (0.81)</td>\n",
              "      <td>large (6.08)</td>\n",
              "      <td>negligible (0.11)</td>\n",
              "      <td>large (1.35)</td>\n",
              "      <td>large (0.87)</td>\n",
              "      <td>large (6.16)</td>\n",
              "      <td>large (3.25)</td>\n",
              "      <td>large (3.24)</td>\n",
              "      <td>large (4.46)</td>\n",
              "      <td>large (4.44)</td>\n",
              "      <td>large (-30.04)</td>\n",
              "      <td>small (0.30)</td>\n",
              "      <td>large (5.63)</td>\n",
              "      <td>large (5.46)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>medium (0.56)</td>\n",
              "      <td>large (1.09)</td>\n",
              "      <td>negligible (0.12)</td>\n",
              "      <td>large (3.86)</td>\n",
              "      <td>medium (0.53)</td>\n",
              "      <td>large (1.53)</td>\n",
              "      <td>negligible (-0.01)</td>\n",
              "      <td>large (4.48)</td>\n",
              "      <td>large (1.06)</td>\n",
              "      <td>large (1.06)</td>\n",
              "      <td>large (1.06)</td>\n",
              "      <td>large (1.06)</td>\n",
              "      <td>large (-5.82)</td>\n",
              "      <td>negligible (0.02)</td>\n",
              "      <td>large (6.89)</td>\n",
              "      <td>large (7.05)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                     M1                                 M2                 \\\n",
              "                 COBYLA             GD              COBYLA             GD   \n",
              "n p                                                                         \n",
              "4 1  negligible (-0.07)   large (1.21)        small (0.21)   large (1.18)   \n",
              "  2   negligible (0.00)   large (4.62)       medium (0.60)  large (18.44)   \n",
              "  3   negligible (0.00)  medium (0.66)       medium (0.57)   large (5.38)   \n",
              "  4        small (0.47)   large (1.17)       medium (0.78)   large (4.53)   \n",
              "  5   negligible (0.00)   large (1.18)        small (0.43)   large (6.40)   \n",
              "5 1      medium (-0.69)   large (1.53)  negligible (-0.16)  medium (0.75)   \n",
              "  2   negligible (0.20)   large (1.19)        large (1.09)   large (3.50)   \n",
              "  3       medium (0.57)   large (1.89)        large (0.81)   large (6.08)   \n",
              "  4       medium (0.56)   large (1.09)   negligible (0.12)   large (3.86)   \n",
              "\n",
              "                     M3                                      M4                \\\n",
              "                 COBYLA                  GD              COBYLA            GD   \n",
              "n p                                                                             \n",
              "4 1  negligible (-0.08)   negligible (0.17)  negligible (-0.08)  large (1.79)   \n",
              "  2       small (-0.46)        large (1.32)       medium (0.58)  large (6.59)   \n",
              "  3       small (-0.40)  negligible (-0.17)       medium (0.73)  large (3.03)   \n",
              "  4      medium (-0.57)       small (-0.38)        large (0.86)  large (4.55)   \n",
              "  5       small (-0.35)      medium (-0.60)        small (0.38)  large (4.62)   \n",
              "5 1      medium (-0.56)       medium (0.59)       small (-0.49)  large (1.54)   \n",
              "  2   negligible (0.01)        large (0.87)        large (1.02)  large (4.13)   \n",
              "  3   negligible (0.11)        large (1.35)        large (0.87)  large (6.16)   \n",
              "  4       medium (0.53)        large (1.53)  negligible (-0.01)  large (4.48)   \n",
              "\n",
              "               M5                           M6                             M7  \\\n",
              "           COBYLA            GD         COBYLA             GD          COBYLA   \n",
              "n p                                                                             \n",
              "4 1  large (6.27)  large (6.02)  large (13.47)  large (13.14)   large (-5.96)   \n",
              "  2  large (1.10)  large (1.09)   large (1.32)   large (1.29)   large (-4.80)   \n",
              "  3  large (0.90)  large (0.90)   large (0.85)   large (0.84)   large (-2.67)   \n",
              "  4  large (4.32)  large (4.32)   large (5.06)   large (5.02)  large (-21.16)   \n",
              "  5  large (1.92)  large (1.92)   large (1.71)   large (1.70)   large (-2.09)   \n",
              "5 1  large (2.75)  large (2.67)   large (4.10)   large (4.06)   large (-5.45)   \n",
              "  2  large (1.62)  large (1.61)   large (2.06)   large (2.04)  large (-14.12)   \n",
              "  3  large (3.25)  large (3.24)   large (4.46)   large (4.44)  large (-30.04)   \n",
              "  4  large (1.06)  large (1.06)   large (1.06)   large (1.06)   large (-5.82)   \n",
              "\n",
              "                                   M8                \n",
              "                     GD        COBYLA            GD  \n",
              "n p                                                  \n",
              "4 1       large (-0.91)  large (4.20)  large (1.59)  \n",
              "  2   negligible (0.17)  large (7.46)  large (7.13)  \n",
              "  3  negligible (-0.04)  large (3.63)  large (4.05)  \n",
              "  4        small (0.34)  large (8.35)  large (7.58)  \n",
              "  5        small (0.36)  large (4.00)  large (4.16)  \n",
              "5 1       large (-1.01)  large (9.41)  large (2.14)  \n",
              "  2       small (-0.44)  large (3.94)  large (3.77)  \n",
              "  3        small (0.30)  large (5.63)  large (5.46)  \n",
              "  4   negligible (0.02)  large (6.89)  large (7.05)  "
            ]
          },
          "execution_count": 96,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "effect_size_combined"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 114,
      "metadata": {},
      "outputs": [],
      "source": [
        "effect_size_directed.astype(float).round(2).to_latex(\"results/effect_size_directed.tex\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 116,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"4\" halign=\"left\">expectation approximation ratio</th>\n",
              "      <th colspan=\"4\" halign=\"left\">groundspace overlap</th>\n",
              "      <th colspan=\"2\" halign=\"left\">groundspace sharpness</th>\n",
              "      <th>...</th>\n",
              "      <th>steps</th>\n",
              "      <th colspan=\"4\" halign=\"left\">path length</th>\n",
              "      <th colspan=\"2\" halign=\"left\">expectation approximation ratio per circuits</th>\n",
              "      <th colspan=\"3\" halign=\"left\">relevance</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>...</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>max</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>count</th>\n",
              "      <th>total</th>\n",
              "      <th>ratio</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>Algorithm</th>\n",
              "      <th>p</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"6\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>0.911088</td>\n",
              "      <td>0.073714</td>\n",
              "      <td>0.858965</td>\n",
              "      <td>0.963212</td>\n",
              "      <td>0.869012</td>\n",
              "      <td>0.145009</td>\n",
              "      <td>0.766475</td>\n",
              "      <td>0.971549</td>\n",
              "      <td>0.294665</td>\n",
              "      <td>0.181596</td>\n",
              "      <td>...</td>\n",
              "      <td>87.0</td>\n",
              "      <td>0.912754</td>\n",
              "      <td>0.036438</td>\n",
              "      <td>0.886988</td>\n",
              "      <td>0.938520</td>\n",
              "      <td>0.000309</td>\n",
              "      <td>-0.000002</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.930390</td>\n",
              "      <td>0.063845</td>\n",
              "      <td>0.885245</td>\n",
              "      <td>0.975536</td>\n",
              "      <td>0.931829</td>\n",
              "      <td>0.071453</td>\n",
              "      <td>0.881304</td>\n",
              "      <td>0.982354</td>\n",
              "      <td>0.227546</td>\n",
              "      <td>0.135601</td>\n",
              "      <td>...</td>\n",
              "      <td>147.0</td>\n",
              "      <td>0.823661</td>\n",
              "      <td>0.014654</td>\n",
              "      <td>0.813299</td>\n",
              "      <td>0.834024</td>\n",
              "      <td>0.000069</td>\n",
              "      <td>-0.000013</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">scipy</th>\n",
              "      <th>1</th>\n",
              "      <td>0.778976</td>\n",
              "      <td>0.112656</td>\n",
              "      <td>0.699316</td>\n",
              "      <td>0.858635</td>\n",
              "      <td>0.766332</td>\n",
              "      <td>0.006235</td>\n",
              "      <td>0.761923</td>\n",
              "      <td>0.770741</td>\n",
              "      <td>0.423131</td>\n",
              "      <td>0.003925</td>\n",
              "      <td>...</td>\n",
              "      <td>24.0</td>\n",
              "      <td>0.131371</td>\n",
              "      <td>0.094648</td>\n",
              "      <td>0.064445</td>\n",
              "      <td>0.198298</td>\n",
              "      <td>0.031795</td>\n",
              "      <td>0.003681</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.930466</td>\n",
              "      <td>0.058679</td>\n",
              "      <td>0.888974</td>\n",
              "      <td>0.971958</td>\n",
              "      <td>0.893380</td>\n",
              "      <td>0.117401</td>\n",
              "      <td>0.810365</td>\n",
              "      <td>0.976395</td>\n",
              "      <td>0.271914</td>\n",
              "      <td>0.169845</td>\n",
              "      <td>...</td>\n",
              "      <td>50.0</td>\n",
              "      <td>0.081390</td>\n",
              "      <td>0.084589</td>\n",
              "      <td>0.021576</td>\n",
              "      <td>0.141203</td>\n",
              "      <td>0.018609</td>\n",
              "      <td>0.000647</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">gradient_descent</th>\n",
              "      <th>1</th>\n",
              "      <td>0.306478</td>\n",
              "      <td>0.432672</td>\n",
              "      <td>0.000533</td>\n",
              "      <td>0.612424</td>\n",
              "      <td>0.358776</td>\n",
              "      <td>0.307027</td>\n",
              "      <td>0.141675</td>\n",
              "      <td>0.575877</td>\n",
              "      <td>0.421463</td>\n",
              "      <td>0.102879</td>\n",
              "      <td>...</td>\n",
              "      <td>59.0</td>\n",
              "      <td>0.230496</td>\n",
              "      <td>0.310428</td>\n",
              "      <td>0.010990</td>\n",
              "      <td>0.450001</td>\n",
              "      <td>0.001293</td>\n",
              "      <td>0.001826</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.652124</td>\n",
              "      <td>0.047679</td>\n",
              "      <td>0.618410</td>\n",
              "      <td>0.685838</td>\n",
              "      <td>0.636691</td>\n",
              "      <td>0.049783</td>\n",
              "      <td>0.601489</td>\n",
              "      <td>0.671892</td>\n",
              "      <td>0.479558</td>\n",
              "      <td>0.014190</td>\n",
              "      <td>...</td>\n",
              "      <td>95.0</td>\n",
              "      <td>0.088509</td>\n",
              "      <td>0.090446</td>\n",
              "      <td>0.024554</td>\n",
              "      <td>0.152464</td>\n",
              "      <td>0.000857</td>\n",
              "      <td>0.000063</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">4</th>\n",
              "      <th rowspan=\"4\" valign=\"top\">tdvp</th>\n",
              "      <th>1</th>\n",
              "      <td>0.555697</td>\n",
              "      <td>0.320149</td>\n",
              "      <td>0.133846</td>\n",
              "      <td>0.963225</td>\n",
              "      <td>0.464344</td>\n",
              "      <td>0.373130</td>\n",
              "      <td>0.019871</td>\n",
              "      <td>0.962394</td>\n",
              "      <td>0.331865</td>\n",
              "      <td>0.164583</td>\n",
              "      <td>...</td>\n",
              "      <td>82.0</td>\n",
              "      <td>0.837968</td>\n",
              "      <td>0.199199</td>\n",
              "      <td>0.434681</td>\n",
              "      <td>0.955152</td>\n",
              "      <td>0.000068</td>\n",
              "      <td>0.000033</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.914538</td>\n",
              "      <td>0.023949</td>\n",
              "      <td>0.873142</td>\n",
              "      <td>0.947408</td>\n",
              "      <td>0.895209</td>\n",
              "      <td>0.049664</td>\n",
              "      <td>0.831052</td>\n",
              "      <td>0.943499</td>\n",
              "      <td>0.297600</td>\n",
              "      <td>0.061860</td>\n",
              "      <td>...</td>\n",
              "      <td>701.0</td>\n",
              "      <td>0.782392</td>\n",
              "      <td>0.116077</td>\n",
              "      <td>0.602306</td>\n",
              "      <td>0.896681</td>\n",
              "      <td>0.000012</td>\n",
              "      <td>-0.000012</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.926812</td>\n",
              "      <td>0.027653</td>\n",
              "      <td>0.895857</td>\n",
              "      <td>0.966206</td>\n",
              "      <td>0.866436</td>\n",
              "      <td>0.068250</td>\n",
              "      <td>0.795922</td>\n",
              "      <td>0.967577</td>\n",
              "      <td>0.324697</td>\n",
              "      <td>0.090627</td>\n",
              "      <td>...</td>\n",
              "      <td>3000.0</td>\n",
              "      <td>0.593010</td>\n",
              "      <td>0.182262</td>\n",
              "      <td>0.372322</td>\n",
              "      <td>0.862888</td>\n",
              "      <td>0.000002</td>\n",
              "      <td>-0.000003</td>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>0.833333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.943432</td>\n",
              "      <td>0.049538</td>\n",
              "      <td>0.869999</td>\n",
              "      <td>0.974706</td>\n",
              "      <td>0.930693</td>\n",
              "      <td>0.062414</td>\n",
              "      <td>0.843536</td>\n",
              "      <td>0.977556</td>\n",
              "      <td>0.232359</td>\n",
              "      <td>0.100608</td>\n",
              "      <td>...</td>\n",
              "      <td>225.0</td>\n",
              "      <td>0.675338</td>\n",
              "      <td>0.099432</td>\n",
              "      <td>0.564874</td>\n",
              "      <td>0.806639</td>\n",
              "      <td>0.000006</td>\n",
              "      <td>-0.000001</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>0.666667</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10 rows × 41 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                     expectation approximation ratio                      \\\n",
              "                                                mean       std       min   \n",
              "n Algorithm        p                                                       \n",
              "3 tdvp             1                        0.911088  0.073714  0.858965   \n",
              "                   2                        0.930390  0.063845  0.885245   \n",
              "  scipy            1                        0.778976  0.112656  0.699316   \n",
              "                   2                        0.930466  0.058679  0.888974   \n",
              "  gradient_descent 1                        0.306478  0.432672  0.000533   \n",
              "                   2                        0.652124  0.047679  0.618410   \n",
              "4 tdvp             1                        0.555697  0.320149  0.133846   \n",
              "                   2                        0.914538  0.023949  0.873142   \n",
              "                   3                        0.926812  0.027653  0.895857   \n",
              "                   4                        0.943432  0.049538  0.869999   \n",
              "\n",
              "                               groundspace overlap                      \\\n",
              "                           max                mean       std       min   \n",
              "n Algorithm        p                                                     \n",
              "3 tdvp             1  0.963212            0.869012  0.145009  0.766475   \n",
              "                   2  0.975536            0.931829  0.071453  0.881304   \n",
              "  scipy            1  0.858635            0.766332  0.006235  0.761923   \n",
              "                   2  0.971958            0.893380  0.117401  0.810365   \n",
              "  gradient_descent 1  0.612424            0.358776  0.307027  0.141675   \n",
              "                   2  0.685838            0.636691  0.049783  0.601489   \n",
              "4 tdvp             1  0.963225            0.464344  0.373130  0.019871   \n",
              "                   2  0.947408            0.895209  0.049664  0.831052   \n",
              "                   3  0.966206            0.866436  0.068250  0.795922   \n",
              "                   4  0.974706            0.930693  0.062414  0.843536   \n",
              "\n",
              "                               groundspace sharpness            ...   steps  \\\n",
              "                           max                  mean       std  ...     max   \n",
              "n Algorithm        p                                            ...           \n",
              "3 tdvp             1  0.971549              0.294665  0.181596  ...    87.0   \n",
              "                   2  0.982354              0.227546  0.135601  ...   147.0   \n",
              "  scipy            1  0.770741              0.423131  0.003925  ...    24.0   \n",
              "                   2  0.976395              0.271914  0.169845  ...    50.0   \n",
              "  gradient_descent 1  0.575877              0.421463  0.102879  ...    59.0   \n",
              "                   2  0.671892              0.479558  0.014190  ...    95.0   \n",
              "4 tdvp             1  0.962394              0.331865  0.164583  ...    82.0   \n",
              "                   2  0.943499              0.297600  0.061860  ...   701.0   \n",
              "                   3  0.967577              0.324697  0.090627  ...  3000.0   \n",
              "                   4  0.977556              0.232359  0.100608  ...   225.0   \n",
              "\n",
              "                     path length                                \\\n",
              "                            mean       std       min       max   \n",
              "n Algorithm        p                                             \n",
              "3 tdvp             1    0.912754  0.036438  0.886988  0.938520   \n",
              "                   2    0.823661  0.014654  0.813299  0.834024   \n",
              "  scipy            1    0.131371  0.094648  0.064445  0.198298   \n",
              "                   2    0.081390  0.084589  0.021576  0.141203   \n",
              "  gradient_descent 1    0.230496  0.310428  0.010990  0.450001   \n",
              "                   2    0.088509  0.090446  0.024554  0.152464   \n",
              "4 tdvp             1    0.837968  0.199199  0.434681  0.955152   \n",
              "                   2    0.782392  0.116077  0.602306  0.896681   \n",
              "                   3    0.593010  0.182262  0.372322  0.862888   \n",
              "                   4    0.675338  0.099432  0.564874  0.806639   \n",
              "\n",
              "                     expectation approximation ratio per circuits            \\\n",
              "                                                             mean       std   \n",
              "n Algorithm        p                                                          \n",
              "3 tdvp             1                                     0.000309 -0.000002   \n",
              "                   2                                     0.000069 -0.000013   \n",
              "  scipy            1                                     0.031795  0.003681   \n",
              "                   2                                     0.018609  0.000647   \n",
              "  gradient_descent 1                                     0.001293  0.001826   \n",
              "                   2                                     0.000857  0.000063   \n",
              "4 tdvp             1                                     0.000068  0.000033   \n",
              "                   2                                     0.000012 -0.000012   \n",
              "                   3                                     0.000002 -0.000003   \n",
              "                   4                                     0.000006 -0.000001   \n",
              "\n",
              "                     relevance                  \n",
              "                         count total     ratio  \n",
              "n Algorithm        p                            \n",
              "3 tdvp             1         2     2       1.0  \n",
              "                   2         2     2       1.0  \n",
              "  scipy            1         2     2       1.0  \n",
              "                   2         2     2       1.0  \n",
              "  gradient_descent 1         2     2       1.0  \n",
              "                   2         2     2       1.0  \n",
              "4 tdvp             1         6     6       1.0  \n",
              "                   2         6     6       1.0  \n",
              "                   3         5     6  0.833333  \n",
              "                   4         4     6  0.666667  \n",
              "\n",
              "[10 rows x 41 columns]"
            ]
          },
          "execution_count": 116,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 118,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>tdvp</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gradient_descent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">expectation approximation ratio</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">groundspace overlap</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">groundspace sharpness</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">approximation ratio</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">circuit count</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">gate count</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">num_fun_calls</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">steps</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">path length</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">expectation approximation ratio per circuits</th>\n",
              "      <th>mean</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"3\" valign=\"top\">relevance</th>\n",
              "      <th>count</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>total</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ratio</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   tdvp scipy gradient_descent\n",
              "expectation approximation ratio              mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "groundspace overlap                          mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "groundspace sharpness                        mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "approximation ratio                          mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "circuit count                                mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "gate count                                   mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "num_fun_calls                                mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "steps                                        mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "path length                                  mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "                                             min    NaN   NaN              NaN\n",
              "                                             max    NaN   NaN              NaN\n",
              "expectation approximation ratio per circuits mean   NaN   NaN              NaN\n",
              "                                             std    NaN   NaN              NaN\n",
              "relevance                                    count  NaN   NaN              NaN\n",
              "                                             total  NaN   NaN              NaN\n",
              "                                             ratio  NaN   NaN              NaN"
            ]
          },
          "execution_count": 118,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats_total = pd.DataFrame(index=stats.columns, columns=['tdvp', 'scipy', 'gradient_descent'])\n",
        "stats_total"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 144,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>approximation ratio</th>\n",
              "      <th>circuit count</th>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <th>gate count</th>\n",
              "      <th>groundspace overlap</th>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <th>num_fun_calls</th>\n",
              "      <th>path length</th>\n",
              "      <th>steps</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th>i</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <th>1</th>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">5</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>5</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>74 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      approximation ratio circuit count expectation approximation ratio  \\\n",
              "n p i                                                                     \n",
              "3 1 0                 NaN           NaN                             NaN   \n",
              "    1                 NaN           NaN                             NaN   \n",
              "  2 0                 NaN           NaN                             NaN   \n",
              "    1                 NaN           NaN                             NaN   \n",
              "4 1 0                 NaN           NaN                             NaN   \n",
              "...                   ...           ...                             ...   \n",
              "5 4 5                 NaN           NaN                             NaN   \n",
              "    6                 NaN           NaN                             NaN   \n",
              "    7                 NaN           NaN                             NaN   \n",
              "    8                 NaN           NaN                             NaN   \n",
              "    9                 NaN           NaN                             NaN   \n",
              "\n",
              "      gate count groundspace overlap groundspace sharpness num_fun_calls  \\\n",
              "n p i                                                                      \n",
              "3 1 0        NaN                 NaN                   NaN           NaN   \n",
              "    1        NaN                 NaN                   NaN           NaN   \n",
              "  2 0        NaN                 NaN                   NaN           NaN   \n",
              "    1        NaN                 NaN                   NaN           NaN   \n",
              "4 1 0        NaN                 NaN                   NaN           NaN   \n",
              "...          ...                 ...                   ...           ...   \n",
              "5 4 5        NaN                 NaN                   NaN           NaN   \n",
              "    6        NaN                 NaN                   NaN           NaN   \n",
              "    7        NaN                 NaN                   NaN           NaN   \n",
              "    8        NaN                 NaN                   NaN           NaN   \n",
              "    9        NaN                 NaN                   NaN           NaN   \n",
              "\n",
              "      path length steps  \n",
              "n p i                    \n",
              "3 1 0         NaN   NaN  \n",
              "    1         NaN   NaN  \n",
              "  2 0         NaN   NaN  \n",
              "    1         NaN   NaN  \n",
              "4 1 0         NaN   NaN  \n",
              "...           ...   ...  \n",
              "5 4 5         NaN   NaN  \n",
              "    6         NaN   NaN  \n",
              "    7         NaN   NaN  \n",
              "    8         NaN   NaN  \n",
              "    9         NaN   NaN  \n",
              "\n",
              "[74 rows x 9 columns]"
            ]
          },
          "execution_count": 144,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tdvp_total = pd.DataFrame(index=data.index, columns=stats.columns.levels[0]).drop(columns=['expectation approximation ratio per circuits', 'relevance'])\n",
        "scipy_total = pd.DataFrame(index=data.index, columns=stats.columns.levels[0]).drop(columns=['expectation approximation ratio per circuits', 'relevance'])\n",
        "gd_total = pd.DataFrame(index=data.index, columns=stats.columns.levels[0]).drop(columns=['expectation approximation ratio per circuits', 'relevance'])\n",
        "tdvp_total"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 170,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"9\" halign=\"left\">tdvp</th>\n",
              "      <th colspan=\"3\" halign=\"left\">scipy</th>\n",
              "      <th colspan=\"9\" halign=\"left\">gradient_descent</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>approximation ratio</th>\n",
              "      <th>circuit count</th>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <th>gate count</th>\n",
              "      <th>groundspace overlap</th>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <th>num_fun_calls</th>\n",
              "      <th>path length</th>\n",
              "      <th>steps</th>\n",
              "      <th>approximation ratio</th>\n",
              "      <th>...</th>\n",
              "      <th>steps</th>\n",
              "      <th>approximation ratio</th>\n",
              "      <th>circuit count</th>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <th>gate count</th>\n",
              "      <th>groundspace overlap</th>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <th>num_fun_calls</th>\n",
              "      <th>path length</th>\n",
              "      <th>steps</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th>i</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>2773.0</td>\n",
              "      <td>0.858965</td>\n",
              "      <td>608040.0</td>\n",
              "      <td>0.766475</td>\n",
              "      <td>0.423073</td>\n",
              "      <td>56</td>\n",
              "      <td>0.93852</td>\n",
              "      <td>77</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>23</td>\n",
              "      <td>1.0</td>\n",
              "      <td>237</td>\n",
              "      <td>0.612424</td>\n",
              "      <td>34032</td>\n",
              "      <td>0.575877</td>\n",
              "      <td>0.494209</td>\n",
              "      <td>None</td>\n",
              "      <td>0.01099</td>\n",
              "      <td>59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>3133.0</td>\n",
              "      <td>0.963212</td>\n",
              "      <td>863448.0</td>\n",
              "      <td>0.971549</td>\n",
              "      <td>0.166258</td>\n",
              "      <td>62</td>\n",
              "      <td>0.886988</td>\n",
              "      <td>87</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>24</td>\n",
              "      <td>-0.333333</td>\n",
              "      <td>237</td>\n",
              "      <td>0.000533</td>\n",
              "      <td>42540</td>\n",
              "      <td>0.141675</td>\n",
              "      <td>0.348716</td>\n",
              "      <td>None</td>\n",
              "      <td>0.450001</td>\n",
              "      <td>59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>10909.0</td>\n",
              "      <td>0.885245</td>\n",
              "      <td>5531016.0</td>\n",
              "      <td>0.881304</td>\n",
              "      <td>0.32343</td>\n",
              "      <td>74</td>\n",
              "      <td>0.813299</td>\n",
              "      <td>101</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>48</td>\n",
              "      <td>1.0</td>\n",
              "      <td>761</td>\n",
              "      <td>0.61841</td>\n",
              "      <td>61614</td>\n",
              "      <td>0.601489</td>\n",
              "      <td>0.489592</td>\n",
              "      <td>None</td>\n",
              "      <td>0.024554</td>\n",
              "      <td>95</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>15877.0</td>\n",
              "      <td>0.975536</td>\n",
              "      <td>9946380.0</td>\n",
              "      <td>0.982354</td>\n",
              "      <td>0.131662</td>\n",
              "      <td>116</td>\n",
              "      <td>0.834024</td>\n",
              "      <td>147</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>50</td>\n",
              "      <td>1.0</td>\n",
              "      <td>761</td>\n",
              "      <td>0.685838</td>\n",
              "      <td>75306</td>\n",
              "      <td>0.671892</td>\n",
              "      <td>0.469524</td>\n",
              "      <td>None</td>\n",
              "      <td>0.152464</td>\n",
              "      <td>95</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <th>1</th>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>6697.0</td>\n",
              "      <td>0.521938</td>\n",
              "      <td>1540540.0</td>\n",
              "      <td>0.45255</td>\n",
              "      <td>0.497743</td>\n",
              "      <td>44</td>\n",
              "      <td>0.898566</td>\n",
              "      <td>62</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>20</td>\n",
              "      <td>1.0</td>\n",
              "      <td>221</td>\n",
              "      <td>0.292312</td>\n",
              "      <td>77528</td>\n",
              "      <td>0.2403</td>\n",
              "      <td>0.427265</td>\n",
              "      <td>None</td>\n",
              "      <td>0.32405</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">5</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>5</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>99</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>5025</td>\n",
              "      <td>-0.167506</td>\n",
              "      <td>1695870</td>\n",
              "      <td>0.072309</td>\n",
              "      <td>0.258999</td>\n",
              "      <td>None</td>\n",
              "      <td>0.069476</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>72</td>\n",
              "      <td>0.571429</td>\n",
              "      <td>5025</td>\n",
              "      <td>-0.113199</td>\n",
              "      <td>1570250</td>\n",
              "      <td>0.023653</td>\n",
              "      <td>0.151964</td>\n",
              "      <td>None</td>\n",
              "      <td>0.06262</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1.0</td>\n",
              "      <td>1135001.0</td>\n",
              "      <td>0.740394</td>\n",
              "      <td>2278223078.0</td>\n",
              "      <td>0.66672</td>\n",
              "      <td>0.471386</td>\n",
              "      <td>380</td>\n",
              "      <td>0.624708</td>\n",
              "      <td>454</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>119</td>\n",
              "      <td>0.510204</td>\n",
              "      <td>5025</td>\n",
              "      <td>0.242827</td>\n",
              "      <td>1444630</td>\n",
              "      <td>0.069378</td>\n",
              "      <td>0.254097</td>\n",
              "      <td>None</td>\n",
              "      <td>0.074261</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>84</td>\n",
              "      <td>-1.285714</td>\n",
              "      <td>5025</td>\n",
              "      <td>-0.425301</td>\n",
              "      <td>1193390</td>\n",
              "      <td>0.074148</td>\n",
              "      <td>0.262011</td>\n",
              "      <td>None</td>\n",
              "      <td>0.045234</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1.0</td>\n",
              "      <td>422501.0</td>\n",
              "      <td>0.880197</td>\n",
              "      <td>614273258.0</td>\n",
              "      <td>0.866424</td>\n",
              "      <td>0.340197</td>\n",
              "      <td>134</td>\n",
              "      <td>0.897946</td>\n",
              "      <td>169</td>\n",
              "      <td>1.0</td>\n",
              "      <td>...</td>\n",
              "      <td>170</td>\n",
              "      <td>1.0</td>\n",
              "      <td>5025</td>\n",
              "      <td>0.150114</td>\n",
              "      <td>1067770</td>\n",
              "      <td>0.140606</td>\n",
              "      <td>0.347615</td>\n",
              "      <td>None</td>\n",
              "      <td>0.04561</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>74 rows × 27 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                     tdvp                                                \\\n",
              "      approximation ratio circuit count expectation approximation ratio   \n",
              "n p i                                                                     \n",
              "3 1 0                 1.0        2773.0                        0.858965   \n",
              "    1                 1.0        3133.0                        0.963212   \n",
              "  2 0                 1.0       10909.0                        0.885245   \n",
              "    1                 1.0       15877.0                        0.975536   \n",
              "4 1 0                 1.0        6697.0                        0.521938   \n",
              "...                   ...           ...                             ...   \n",
              "5 4 5                 NaN           NaN                             NaN   \n",
              "    6                 NaN           NaN                             NaN   \n",
              "    7                 1.0     1135001.0                        0.740394   \n",
              "    8                 NaN           NaN                             NaN   \n",
              "    9                 1.0      422501.0                        0.880197   \n",
              "\n",
              "                                                                             \\\n",
              "         gate count groundspace overlap groundspace sharpness num_fun_calls   \n",
              "n p i                                                                         \n",
              "3 1 0      608040.0            0.766475              0.423073            56   \n",
              "    1      863448.0            0.971549              0.166258            62   \n",
              "  2 0     5531016.0            0.881304               0.32343            74   \n",
              "    1     9946380.0            0.982354              0.131662           116   \n",
              "4 1 0     1540540.0             0.45255              0.497743            44   \n",
              "...             ...                 ...                   ...           ...   \n",
              "5 4 5           NaN                 NaN                   NaN             0   \n",
              "    6           NaN                 NaN                   NaN             0   \n",
              "    7  2278223078.0             0.66672              0.471386           380   \n",
              "    8           NaN                 NaN                   NaN             0   \n",
              "    9   614273258.0            0.866424              0.340197           134   \n",
              "\n",
              "                                      scipy  ...          gradient_descent  \\\n",
              "      path length steps approximation ratio  ... steps approximation ratio   \n",
              "n p i                                        ...                             \n",
              "3 1 0     0.93852    77                 1.0  ...    23                 1.0   \n",
              "    1    0.886988    87                 1.0  ...    24           -0.333333   \n",
              "  2 0    0.813299   101                 1.0  ...    48                 1.0   \n",
              "    1    0.834024   147                 1.0  ...    50                 1.0   \n",
              "4 1 0    0.898566    62                 1.0  ...    20                 1.0   \n",
              "...           ...   ...                 ...  ...   ...                 ...   \n",
              "5 4 5         NaN  None                 1.0  ...    99           -1.285714   \n",
              "    6         NaN  None                 1.0  ...    72            0.571429   \n",
              "    7    0.624708   454                 1.0  ...   119            0.510204   \n",
              "    8         NaN  None                 1.0  ...    84           -1.285714   \n",
              "    9    0.897946   169                 1.0  ...   170                 1.0   \n",
              "\n",
              "                                                                \\\n",
              "      circuit count expectation approximation ratio gate count   \n",
              "n p i                                                            \n",
              "3 1 0           237                        0.612424      34032   \n",
              "    1           237                        0.000533      42540   \n",
              "  2 0           761                         0.61841      61614   \n",
              "    1           761                        0.685838      75306   \n",
              "4 1 0           221                        0.292312      77528   \n",
              "...             ...                             ...        ...   \n",
              "5 4 5          5025                       -0.167506    1695870   \n",
              "    6          5025                       -0.113199    1570250   \n",
              "    7          5025                        0.242827    1444630   \n",
              "    8          5025                       -0.425301    1193390   \n",
              "    9          5025                        0.150114    1067770   \n",
              "\n",
              "                                                                           \\\n",
              "      groundspace overlap groundspace sharpness num_fun_calls path length   \n",
              "n p i                                                                       \n",
              "3 1 0            0.575877              0.494209          None     0.01099   \n",
              "    1            0.141675              0.348716          None    0.450001   \n",
              "  2 0            0.601489              0.489592          None    0.024554   \n",
              "    1            0.671892              0.469524          None    0.152464   \n",
              "4 1 0              0.2403              0.427265          None     0.32405   \n",
              "...                   ...                   ...           ...         ...   \n",
              "5 4 5            0.072309              0.258999          None    0.069476   \n",
              "    6            0.023653              0.151964          None     0.06262   \n",
              "    7            0.069378              0.254097          None    0.074261   \n",
              "    8            0.074148              0.262011          None    0.045234   \n",
              "    9            0.140606              0.347615          None     0.04561   \n",
              "\n",
              "             \n",
              "      steps  \n",
              "n p i        \n",
              "3 1 0    59  \n",
              "    1    59  \n",
              "  2 0    95  \n",
              "    1    95  \n",
              "4 1 0    55  \n",
              "...     ...  \n",
              "5 4 5   314  \n",
              "    6   314  \n",
              "    7   314  \n",
              "    8   314  \n",
              "    9   314  \n",
              "\n",
              "[74 rows x 27 columns]"
            ]
          },
          "execution_count": 170,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data_total = pd.concat([tdvp_total, scipy_total, gd_total], keys=['tdvp', 'scipy', 'gradient_descent'], axis=1)\n",
        "data_total"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 230,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>approximation ratio</th>\n",
              "      <th>circuit count</th>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <th>expectation approximation ratio per circuits</th>\n",
              "      <th>gate count</th>\n",
              "      <th>groundspace overlap</th>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <th>num_fun_calls</th>\n",
              "      <th>path length</th>\n",
              "      <th>steps</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th>i</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>237</td>\n",
              "      <td>0.612424</td>\n",
              "      <td>0.002584</td>\n",
              "      <td>34032</td>\n",
              "      <td>0.575877</td>\n",
              "      <td>0.494209</td>\n",
              "      <td>None</td>\n",
              "      <td>0.01099</td>\n",
              "      <td>59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.333333</td>\n",
              "      <td>237</td>\n",
              "      <td>0.000533</td>\n",
              "      <td>0.000002</td>\n",
              "      <td>42540</td>\n",
              "      <td>0.141675</td>\n",
              "      <td>0.348716</td>\n",
              "      <td>None</td>\n",
              "      <td>0.450001</td>\n",
              "      <td>59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>761</td>\n",
              "      <td>0.61841</td>\n",
              "      <td>0.000813</td>\n",
              "      <td>61614</td>\n",
              "      <td>0.601489</td>\n",
              "      <td>0.489592</td>\n",
              "      <td>None</td>\n",
              "      <td>0.024554</td>\n",
              "      <td>95</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>761</td>\n",
              "      <td>0.685838</td>\n",
              "      <td>0.000901</td>\n",
              "      <td>75306</td>\n",
              "      <td>0.671892</td>\n",
              "      <td>0.469524</td>\n",
              "      <td>None</td>\n",
              "      <td>0.152464</td>\n",
              "      <td>95</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <th>1</th>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>221</td>\n",
              "      <td>0.292312</td>\n",
              "      <td>0.001323</td>\n",
              "      <td>77528</td>\n",
              "      <td>0.2403</td>\n",
              "      <td>0.427265</td>\n",
              "      <td>None</td>\n",
              "      <td>0.32405</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">5</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>5</th>\n",
              "      <td>-1.285714</td>\n",
              "      <td>5025</td>\n",
              "      <td>-0.167506</td>\n",
              "      <td>-0.000033</td>\n",
              "      <td>1695870</td>\n",
              "      <td>0.072309</td>\n",
              "      <td>0.258999</td>\n",
              "      <td>None</td>\n",
              "      <td>0.069476</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0.571429</td>\n",
              "      <td>5025</td>\n",
              "      <td>-0.113199</td>\n",
              "      <td>-0.000023</td>\n",
              "      <td>1570250</td>\n",
              "      <td>0.023653</td>\n",
              "      <td>0.151964</td>\n",
              "      <td>None</td>\n",
              "      <td>0.06262</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0.510204</td>\n",
              "      <td>5025</td>\n",
              "      <td>0.242827</td>\n",
              "      <td>0.000048</td>\n",
              "      <td>1444630</td>\n",
              "      <td>0.069378</td>\n",
              "      <td>0.254097</td>\n",
              "      <td>None</td>\n",
              "      <td>0.074261</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>-1.285714</td>\n",
              "      <td>5025</td>\n",
              "      <td>-0.425301</td>\n",
              "      <td>-0.000085</td>\n",
              "      <td>1193390</td>\n",
              "      <td>0.074148</td>\n",
              "      <td>0.262011</td>\n",
              "      <td>None</td>\n",
              "      <td>0.045234</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1.0</td>\n",
              "      <td>5025</td>\n",
              "      <td>0.150114</td>\n",
              "      <td>0.00003</td>\n",
              "      <td>1067770</td>\n",
              "      <td>0.140606</td>\n",
              "      <td>0.347615</td>\n",
              "      <td>None</td>\n",
              "      <td>0.04561</td>\n",
              "      <td>314</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>74 rows × 10 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      approximation ratio circuit count expectation approximation ratio  \\\n",
              "n p i                                                                     \n",
              "3 1 0                 1.0           237                        0.612424   \n",
              "    1           -0.333333           237                        0.000533   \n",
              "  2 0                 1.0           761                         0.61841   \n",
              "    1                 1.0           761                        0.685838   \n",
              "4 1 0                 1.0           221                        0.292312   \n",
              "...                   ...           ...                             ...   \n",
              "5 4 5           -1.285714          5025                       -0.167506   \n",
              "    6            0.571429          5025                       -0.113199   \n",
              "    7            0.510204          5025                        0.242827   \n",
              "    8           -1.285714          5025                       -0.425301   \n",
              "    9                 1.0          5025                        0.150114   \n",
              "\n",
              "      expectation approximation ratio per circuits gate count  \\\n",
              "n p i                                                           \n",
              "3 1 0                                     0.002584      34032   \n",
              "    1                                     0.000002      42540   \n",
              "  2 0                                     0.000813      61614   \n",
              "    1                                     0.000901      75306   \n",
              "4 1 0                                     0.001323      77528   \n",
              "...                                            ...        ...   \n",
              "5 4 5                                    -0.000033    1695870   \n",
              "    6                                    -0.000023    1570250   \n",
              "    7                                     0.000048    1444630   \n",
              "    8                                    -0.000085    1193390   \n",
              "    9                                      0.00003    1067770   \n",
              "\n",
              "      groundspace overlap groundspace sharpness num_fun_calls path length  \\\n",
              "n p i                                                                       \n",
              "3 1 0            0.575877              0.494209          None     0.01099   \n",
              "    1            0.141675              0.348716          None    0.450001   \n",
              "  2 0            0.601489              0.489592          None    0.024554   \n",
              "    1            0.671892              0.469524          None    0.152464   \n",
              "4 1 0              0.2403              0.427265          None     0.32405   \n",
              "...                   ...                   ...           ...         ...   \n",
              "5 4 5            0.072309              0.258999          None    0.069476   \n",
              "    6            0.023653              0.151964          None     0.06262   \n",
              "    7            0.069378              0.254097          None    0.074261   \n",
              "    8            0.074148              0.262011          None    0.045234   \n",
              "    9            0.140606              0.347615          None     0.04561   \n",
              "\n",
              "      steps  \n",
              "n p i        \n",
              "3 1 0    59  \n",
              "    1    59  \n",
              "  2 0    95  \n",
              "    1    95  \n",
              "4 1 0    55  \n",
              "...     ...  \n",
              "5 4 5   314  \n",
              "    6   314  \n",
              "    7   314  \n",
              "    8   314  \n",
              "    9   314  \n",
              "\n",
              "[74 rows x 10 columns]"
            ]
          },
          "execution_count": 230,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data_total['gradient_descent']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 172,
      "metadata": {},
      "outputs": [],
      "source": [
        "for alg in data_total.columns.levels[0]:\n",
        "    ser = data_total[alg]['expectation approximation ratio']/data_total[alg]['circuit count']\n",
        "    data_total[alg, 'expectation approximation ratio per circuits'] = ser"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 178,
      "metadata": {},
      "outputs": [],
      "source": [
        "data_total.sort_index(axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 189,
      "metadata": {},
      "outputs": [],
      "source": [
        "with open(\"results/data_total.p\",'rb') as file:\n",
        "    data_total = pickle.load(file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 195,
      "metadata": {},
      "outputs": [],
      "source": [
        "for meas in stats_total.index.levels[0]:\n",
        "    for alg in stats_total.columns:\n",
        "        if meas != 'relevance':\n",
        "            stats_total.loc[meas, 'mean'][alg] = data_total[alg, meas].mean()\n",
        "            stats_total.loc[meas, 'std'][alg] = data_total[alg, meas].std()\n",
        "            if meas != 'expectation approximation ratio per circuits':\n",
        "                stats_total.loc[meas, 'min'][alg] = data_total[alg, meas].min()\n",
        "                stats_total.loc[meas, 'max'][alg] = data_total[alg, meas].max()\n",
        "            \n",
        "        \n",
        "            "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 231,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tdvp\n",
            "scipy\n",
            "gradient_descent\n"
          ]
        }
      ],
      "source": [
        "for alg in stats_total.columns:\n",
        "    stats_total.loc['relevance', 'count'][alg] = len(data_total[data_total[alg,'gate count'].apply(lambda x: not np.isnan(x))])\n",
        "    stats_total.loc['relevance', 'total'][alg] = len(data_total[alg,'gate count'])\n",
        "    stats_total.loc['relevance', 'ratio'][alg] = stats_total.loc['relevance', 'count'][alg]/stats_total.loc['relevance', 'total'][alg]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 229,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "74"
            ]
          },
          "execution_count": 229,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(data_total[data_total['gradient_descent','gate count'].apply(lambda x: not np.isnan(x))])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 278,
      "metadata": {},
      "outputs": [],
      "source": [
        "stats_total.rename({'gradient_descent':'gd'}, axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 301,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>tdvp</th>\n",
              "      <th>scipy</th>\n",
              "      <th>gd</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">expectation approximation ratio</th>\n",
              "      <th>mean</th>\n",
              "      <td>0.766499</td>\n",
              "      <td>0.7323</td>\n",
              "      <td>0.034207</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.266474</td>\n",
              "      <td>0.223843</td>\n",
              "      <td>0.226152</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>-0.001885</td>\n",
              "      <td>0.096577</td>\n",
              "      <td>-0.657712</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.999978</td>\n",
              "      <td>0.995812</td>\n",
              "      <td>0.685838</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">groundspace overlap</th>\n",
              "      <th>mean</th>\n",
              "      <td>0.688814</td>\n",
              "      <td>0.591318</td>\n",
              "      <td>0.120222</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.322811</td>\n",
              "      <td>0.334467</td>\n",
              "      <td>0.151143</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.001621</td>\n",
              "      <td>0.000165</td>\n",
              "      <td>0.001355</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.999983</td>\n",
              "      <td>0.99674</td>\n",
              "      <td>0.671892</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">groundspace sharpness</th>\n",
              "      <th>mean</th>\n",
              "      <td>0.3086</td>\n",
              "      <td>0.335777</td>\n",
              "      <td>0.257976</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.130104</td>\n",
              "      <td>0.137158</td>\n",
              "      <td>0.130039</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.004178</td>\n",
              "      <td>0.012831</td>\n",
              "      <td>0.036785</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.499365</td>\n",
              "      <td>0.49996</td>\n",
              "      <td>0.4978</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">approximation ratio</th>\n",
              "      <th>mean</th>\n",
              "      <td>0.892778</td>\n",
              "      <td>0.926917</td>\n",
              "      <td>0.115799</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.274183</td>\n",
              "      <td>0.162662</td>\n",
              "      <td>0.728616</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>-0.190476</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>-1.285714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">circuit count</th>\n",
              "      <th>mean</th>\n",
              "      <td>351638.566667</td>\n",
              "      <td>76.567568</td>\n",
              "      <td>2498.405405</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>905807.622728</td>\n",
              "      <td>53.09131</td>\n",
              "      <td>2469.136829</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>2773.0</td>\n",
              "      <td>17</td>\n",
              "      <td>221</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>6257501.0</td>\n",
              "      <td>336</td>\n",
              "      <td>7401</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">gate count</th>\n",
              "      <th>mean</th>\n",
              "      <td>567492444.233333</td>\n",
              "      <td>12412.540541</td>\n",
              "      <td>587773.567568</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>1684716397.003699</td>\n",
              "      <td>8434.722542</td>\n",
              "      <td>453406.164104</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>608040.0</td>\n",
              "      <td>1152</td>\n",
              "      <td>34032</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>11406151186.0</td>\n",
              "      <td>42420</td>\n",
              "      <td>1821490</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">num_fun_calls</th>\n",
              "      <th>mean</th>\n",
              "      <td>195.27027</td>\n",
              "      <td>75.567568</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>420.278391</td>\n",
              "      <td>53.09131</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0</td>\n",
              "      <td>16</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>2588</td>\n",
              "      <td>335</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">steps</th>\n",
              "      <th>mean</th>\n",
              "      <td>290.333333</td>\n",
              "      <td>75.567568</td>\n",
              "      <td>195.108108</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>524.700937</td>\n",
              "      <td>53.09131</td>\n",
              "      <td>147.691605</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>36</td>\n",
              "      <td>16</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>3000</td>\n",
              "      <td>335</td>\n",
              "      <td>548</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">path length</th>\n",
              "      <th>mean</th>\n",
              "      <td>0.742549</td>\n",
              "      <td>0.103798</td>\n",
              "      <td>0.160914</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.178936</td>\n",
              "      <td>0.054821</td>\n",
              "      <td>0.194875</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.325542</td>\n",
              "      <td>0.018789</td>\n",
              "      <td>0.01099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.999191</td>\n",
              "      <td>0.238016</td>\n",
              "      <td>0.952922</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">expectation approximation ratio per circuits</th>\n",
              "      <th>mean</th>\n",
              "      <td>0.000027</td>\n",
              "      <td>0.013232</td>\n",
              "      <td>0.000117</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.000058</td>\n",
              "      <td>0.008373</td>\n",
              "      <td>0.000387</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"3\" valign=\"top\">relevance</th>\n",
              "      <th>count</th>\n",
              "      <td>60</td>\n",
              "      <td>74</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>total</th>\n",
              "      <td>74</td>\n",
              "      <td>74</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ratio</th>\n",
              "      <td>0.810811</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                 tdvp  \\\n",
              "expectation approximation ratio              mean            0.766499   \n",
              "                                             std             0.266474   \n",
              "                                             min            -0.001885   \n",
              "                                             max             0.999978   \n",
              "groundspace overlap                          mean            0.688814   \n",
              "                                             std             0.322811   \n",
              "                                             min             0.001621   \n",
              "                                             max             0.999983   \n",
              "groundspace sharpness                        mean              0.3086   \n",
              "                                             std             0.130104   \n",
              "                                             min             0.004178   \n",
              "                                             max             0.499365   \n",
              "approximation ratio                          mean            0.892778   \n",
              "                                             std             0.274183   \n",
              "                                             min            -0.190476   \n",
              "                                             max                  1.0   \n",
              "circuit count                                mean       351638.566667   \n",
              "                                             std        905807.622728   \n",
              "                                             min               2773.0   \n",
              "                                             max            6257501.0   \n",
              "gate count                                   mean    567492444.233333   \n",
              "                                             std    1684716397.003699   \n",
              "                                             min             608040.0   \n",
              "                                             max        11406151186.0   \n",
              "num_fun_calls                                mean           195.27027   \n",
              "                                             std           420.278391   \n",
              "                                             min                    0   \n",
              "                                             max                 2588   \n",
              "steps                                        mean          290.333333   \n",
              "                                             std           524.700937   \n",
              "                                             min                   36   \n",
              "                                             max                 3000   \n",
              "path length                                  mean            0.742549   \n",
              "                                             std             0.178936   \n",
              "                                             min             0.325542   \n",
              "                                             max             0.999191   \n",
              "expectation approximation ratio per circuits mean            0.000027   \n",
              "                                             std             0.000058   \n",
              "relevance                                    count                 60   \n",
              "                                             total                 74   \n",
              "                                             ratio           0.810811   \n",
              "\n",
              "                                                           scipy  \\\n",
              "expectation approximation ratio              mean         0.7323   \n",
              "                                             std        0.223843   \n",
              "                                             min        0.096577   \n",
              "                                             max        0.995812   \n",
              "groundspace overlap                          mean       0.591318   \n",
              "                                             std        0.334467   \n",
              "                                             min        0.000165   \n",
              "                                             max         0.99674   \n",
              "groundspace sharpness                        mean       0.335777   \n",
              "                                             std        0.137158   \n",
              "                                             min        0.012831   \n",
              "                                             max         0.49996   \n",
              "approximation ratio                          mean       0.926917   \n",
              "                                             std        0.162662   \n",
              "                                             min        0.428571   \n",
              "                                             max             1.0   \n",
              "circuit count                                mean      76.567568   \n",
              "                                             std        53.09131   \n",
              "                                             min              17   \n",
              "                                             max             336   \n",
              "gate count                                   mean   12412.540541   \n",
              "                                             std     8434.722542   \n",
              "                                             min            1152   \n",
              "                                             max           42420   \n",
              "num_fun_calls                                mean      75.567568   \n",
              "                                             std        53.09131   \n",
              "                                             min              16   \n",
              "                                             max             335   \n",
              "steps                                        mean      75.567568   \n",
              "                                             std        53.09131   \n",
              "                                             min              16   \n",
              "                                             max             335   \n",
              "path length                                  mean       0.103798   \n",
              "                                             std        0.054821   \n",
              "                                             min        0.018789   \n",
              "                                             max        0.238016   \n",
              "expectation approximation ratio per circuits mean       0.013232   \n",
              "                                             std        0.008373   \n",
              "relevance                                    count            74   \n",
              "                                             total            74   \n",
              "                                             ratio           1.0   \n",
              "\n",
              "                                                               gd  \n",
              "expectation approximation ratio              mean        0.034207  \n",
              "                                             std         0.226152  \n",
              "                                             min        -0.657712  \n",
              "                                             max         0.685838  \n",
              "groundspace overlap                          mean        0.120222  \n",
              "                                             std         0.151143  \n",
              "                                             min         0.001355  \n",
              "                                             max         0.671892  \n",
              "groundspace sharpness                        mean        0.257976  \n",
              "                                             std         0.130039  \n",
              "                                             min         0.036785  \n",
              "                                             max           0.4978  \n",
              "approximation ratio                          mean        0.115799  \n",
              "                                             std         0.728616  \n",
              "                                             min        -1.285714  \n",
              "                                             max              1.0  \n",
              "circuit count                                mean     2498.405405  \n",
              "                                             std      2469.136829  \n",
              "                                             min              221  \n",
              "                                             max             7401  \n",
              "gate count                                   mean   587773.567568  \n",
              "                                             std    453406.164104  \n",
              "                                             min            34032  \n",
              "                                             max          1821490  \n",
              "num_fun_calls                                mean             NaN  \n",
              "                                             std              NaN  \n",
              "                                             min              NaN  \n",
              "                                             max              NaN  \n",
              "steps                                        mean      195.108108  \n",
              "                                             std       147.691605  \n",
              "                                             min               55  \n",
              "                                             max              548  \n",
              "path length                                  mean        0.160914  \n",
              "                                             std         0.194875  \n",
              "                                             min          0.01099  \n",
              "                                             max         0.952922  \n",
              "expectation approximation ratio per circuits mean        0.000117  \n",
              "                                             std         0.000387  \n",
              "relevance                                    count             74  \n",
              "                                             total             74  \n",
              "                                             ratio            1.0  "
            ]
          },
          "execution_count": 301,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "stats_total"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 279,
      "metadata": {},
      "outputs": [],
      "source": [
        "with open(\"results/stats_total.p\",'wb') as file:\n",
        "    pickle.dump(stats_total,file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 289,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>scipy</th>\n",
              "      <th>gd</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>approximation ratio</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>circuit count</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>expectation approximation ratio per circuits</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gate count</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>groundspace overlap</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>path length</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                             scipy   gd\n",
              "approximation ratio                            NaN  NaN\n",
              "circuit count                                  NaN  NaN\n",
              "expectation approximation ratio                NaN  NaN\n",
              "expectation approximation ratio per circuits   NaN  NaN\n",
              "gate count                                     NaN  NaN\n",
              "groundspace overlap                            NaN  NaN\n",
              "groundspace sharpness                          NaN  NaN\n",
              "path length                                    NaN  NaN"
            ]
          },
          "execution_count": 289,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "total_effect_size= pd.DataFrame(index=stats_total.index.levels[0], columns=['scipy', 'gd'])\n",
        "total_effect_size.drop(['steps', 'num_fun_calls', 'relevance'],inplace=True)\n",
        "total_effect_size"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 280,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2.9719709009317032"
            ]
          },
          "execution_count": 280,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "hedges_g(\n",
        "    stats_total[\"tdvp\"][\"expectation approximation ratio\"]['mean'],\n",
        "    stats_total[\"gd\"][\"expectation approximation ratio\"]['mean'],\n",
        "    stats_total['tdvp']['expectation approximation ratio']['std'],stats_total['gd']['expectation approximation ratio']['std'],\n",
        "    stats_total['tdvp']['relevance', 'count'],stats_total['gd']['relevance', 'count']\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 299,
      "metadata": {},
      "outputs": [],
      "source": [
        "for meas in total_effect_size.index:\n",
        "    for alg in [\"scipy\", \"gd\"]:\n",
        "        total_effect_size.loc[meas, alg] = hedges_g(\n",
        "            m1=stats_total[\"tdvp\"][meas]['mean'],\n",
        "            m2=stats_total[alg][meas]['mean'],\n",
        "            s1=stats_total['tdvp'][meas]['std'],\n",
        "            s2=stats_total[alg][meas]['std'],\n",
        "            n1=stats_total['tdvp']['relevance', 'count'],\n",
        "            n2=stats_total[alg]['relevance', 'count']\n",
        "        )\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 305,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>scipy</th>\n",
              "      <th>gd</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>approximation ratio</th>\n",
              "      <td>0.154559</td>\n",
              "      <td>1.350597</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>circuit count</th>\n",
              "      <td>0.577227</td>\n",
              "      <td>0.573248</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <td>0.139464</td>\n",
              "      <td>2.971971</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>expectation approximation ratio per circuits</th>\n",
              "      <td>2.108684</td>\n",
              "      <td>0.309666</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gate count</th>\n",
              "      <td>0.500962</td>\n",
              "      <td>0.500454</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>groundspace overlap</th>\n",
              "      <td>0.294375</td>\n",
              "      <td>2.323378</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <td>0.201578</td>\n",
              "      <td>0.387002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>path length</th>\n",
              "      <td>5.025245</td>\n",
              "      <td>3.077535</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 scipy        gd\n",
              "approximation ratio                           0.154559  1.350597\n",
              "circuit count                                 0.577227  0.573248\n",
              "expectation approximation ratio               0.139464  2.971971\n",
              "expectation approximation ratio per circuits  2.108684  0.309666\n",
              "gate count                                    0.500962  0.500454\n",
              "groundspace overlap                           0.294375  2.323378\n",
              "groundspace sharpness                         0.201578  0.387002\n",
              "path length                                   5.025245  3.077535"
            ]
          },
          "execution_count": 305,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "total_effect_size.apply(np.abs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 307,
      "metadata": {},
      "outputs": [],
      "source": [
        "total_effect_size_category = total_effect_size.copy()\n",
        "for idx in total_effect_size_category.index:\n",
        "    for alg in total_effect_size_category.loc[idx].index:\n",
        "        if total_effect_size_category.loc[idx, alg] < 0.2:\n",
        "            total_effect_size_category.loc[idx, alg] = \"negligible\"\n",
        "        elif total_effect_size_category.loc[idx, alg] < 0.8:\n",
        "            total_effect_size_category.loc[idx, alg] = \"small\"\n",
        "        elif total_effect_size_category.loc[idx, alg] < 1.2:\n",
        "            total_effect_size_category.loc[idx, alg] = \"medium\"\n",
        "        else:\n",
        "            total_effect_size_category.loc[idx, alg] = \"large\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 312,
      "metadata": {},
      "outputs": [],
      "source": [
        "total_effect_size_combined = total_effect_size_category.copy()\n",
        "for meas in total_effect_size_combined.index:\n",
        "    for comp in total_effect_size_combined.loc[idx].index:\n",
        "        total_effect_size_combined.loc[meas][comp] = total_effect_size_category.loc[meas][comp] + \" (\" + \"{:.2f}\".format(total_effect_size.loc[meas][comp])+\")\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 316,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>scipy</th>\n",
              "      <th>gd</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>approximation ratio</th>\n",
              "      <td>negligible (-0.15)</td>\n",
              "      <td>large (1.35)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>circuit count</th>\n",
              "      <td>small (0.58)</td>\n",
              "      <td>small (0.57)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <td>negligible (0.14)</td>\n",
              "      <td>large (2.97)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>expectation approximation ratio per circuits</th>\n",
              "      <td>negligible (-2.11)</td>\n",
              "      <td>negligible (-0.31)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gate count</th>\n",
              "      <td>small (0.50)</td>\n",
              "      <td>small (0.50)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>groundspace overlap</th>\n",
              "      <td>small (0.29)</td>\n",
              "      <td>large (2.32)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <td>negligible (-0.20)</td>\n",
              "      <td>small (0.39)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>path length</th>\n",
              "      <td>NA</td>\n",
              "      <td>large (3.08)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                           scipy  \\\n",
              "approximation ratio                           negligible (-0.15)   \n",
              "circuit count                                       small (0.58)   \n",
              "expectation approximation ratio                negligible (0.14)   \n",
              "expectation approximation ratio per circuits  negligible (-2.11)   \n",
              "gate count                                          small (0.50)   \n",
              "groundspace overlap                                 small (0.29)   \n",
              "groundspace sharpness                         negligible (-0.20)   \n",
              "path length                                                   NA   \n",
              "\n",
              "                                                              gd  \n",
              "approximation ratio                                 large (1.35)  \n",
              "circuit count                                       small (0.57)  \n",
              "expectation approximation ratio                     large (2.97)  \n",
              "expectation approximation ratio per circuits  negligible (-0.31)  \n",
              "gate count                                          small (0.50)  \n",
              "groundspace overlap                                 large (2.32)  \n",
              "groundspace sharpness                               small (0.39)  \n",
              "path length                                         large (3.08)  "
            ]
          },
          "execution_count": 316,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "total_effect_size_combined"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 317,
      "metadata": {},
      "outputs": [],
      "source": [
        "with open(\"results/total_effect_size_combined.p\",'wb') as file:\n",
        "    pickle.dump(total_effect_size_combined,file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 325,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([0.9385196653061596, 0.886988498178056, 0.8132991915420401,\n",
              "       0.834023546460368, 0.898566465021044, 0.9098216581203107,\n",
              "       0.9551522874502899, 0.8884281807732192, 0.4346812504997844,\n",
              "       0.9411590552586566, 0.6818773551482376, 0.8966807123533533,\n",
              "       0.8592013964597753, 0.8577096240528604, 0.602306232922257,\n",
              "       0.7965784688765338, 0.8628877880873481, 0.556574013390177,\n",
              "       0.6571032231743528, 0.5161632105005212, 0.3723222468075711, nan,\n",
              "       0.5648741229461829, 0.6642272985245559, 0.6656117225122345, nan,\n",
              "       0.806639148451193, nan, nan, 0.3255419493157514,\n",
              "       0.6943631641123666, nan, 0.5651577563331852, nan,\n",
              "       0.9380418836840865, 0.8622280537077172, 0.8868133234266093,\n",
              "       0.9991908287960958, 0.9987348932615168, 0.9873462055465452,\n",
              "       0.8744911256946677, 0.7358668422213904, 0.8933445582257235,\n",
              "       0.9855383134883111, 0.6877881897748995, 0.4990241933554088,\n",
              "       0.5744335625775114, nan, 0.7272060983125394, 0.4811458890969179,\n",
              "       0.35535876525635807, 0.584275827986642, 0.8295080232856704,\n",
              "       0.8372255907137928, 0.7894436257281927, 0.8115830451854306, nan,\n",
              "       nan, 0.6999217619131508, 0.400375758393374, nan,\n",
              "       0.6204278200774638, 0.840073976264369, 0.8350373243045957,\n",
              "       0.9021986201829658, 0.7729188164335282, 0.5964669396581374, nan,\n",
              "       0.5778412955447809, nan, nan, 0.6247081368916103, nan,\n",
              "       0.8979464648226597], dtype=object)"
            ]
          },
          "execution_count": 325,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "np.array(data_total['tdvp']['path length'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 342,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th colspan=\"10\" halign=\"left\">gradient_descent</th>\n",
              "      <th>...</th>\n",
              "      <th colspan=\"10\" halign=\"left\">tdvp</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>approximation ratio</th>\n",
              "      <th>circuit count</th>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <th>expectation approximation ratio per circuits</th>\n",
              "      <th>gate count</th>\n",
              "      <th>groundspace overlap</th>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <th>num_fun_calls</th>\n",
              "      <th>path length</th>\n",
              "      <th>steps</th>\n",
              "      <th>...</th>\n",
              "      <th>approximation ratio</th>\n",
              "      <th>circuit count</th>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <th>expectation approximation ratio per circuits</th>\n",
              "      <th>gate count</th>\n",
              "      <th>groundspace overlap</th>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <th>num_fun_calls</th>\n",
              "      <th>path length</th>\n",
              "      <th>steps</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n</th>\n",
              "      <th>p</th>\n",
              "      <th>i</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"4\" valign=\"top\">3</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>0</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>237.0</td>\n",
              "      <td>0.612424</td>\n",
              "      <td>0.002584</td>\n",
              "      <td>34032.0</td>\n",
              "      <td>0.575877</td>\n",
              "      <td>0.494209</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.010990</td>\n",
              "      <td>59.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2773.0</td>\n",
              "      <td>0.858965</td>\n",
              "      <td>3.097601e-04</td>\n",
              "      <td>6.080400e+05</td>\n",
              "      <td>0.766475</td>\n",
              "      <td>0.423073</td>\n",
              "      <td>56.0</td>\n",
              "      <td>0.938520</td>\n",
              "      <td>77.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.333333</td>\n",
              "      <td>237.0</td>\n",
              "      <td>0.000533</td>\n",
              "      <td>0.000002</td>\n",
              "      <td>42540.0</td>\n",
              "      <td>0.141675</td>\n",
              "      <td>0.348716</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.450001</td>\n",
              "      <td>59.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>3133.0</td>\n",
              "      <td>0.963212</td>\n",
              "      <td>3.074409e-04</td>\n",
              "      <td>8.634480e+05</td>\n",
              "      <td>0.971549</td>\n",
              "      <td>0.166258</td>\n",
              "      <td>62.0</td>\n",
              "      <td>0.886988</td>\n",
              "      <td>87.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>0</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>761.0</td>\n",
              "      <td>0.618410</td>\n",
              "      <td>0.000813</td>\n",
              "      <td>61614.0</td>\n",
              "      <td>0.601489</td>\n",
              "      <td>0.489592</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.024554</td>\n",
              "      <td>95.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>10909.0</td>\n",
              "      <td>0.885245</td>\n",
              "      <td>8.114813e-05</td>\n",
              "      <td>5.531016e+06</td>\n",
              "      <td>0.881304</td>\n",
              "      <td>0.323430</td>\n",
              "      <td>74.0</td>\n",
              "      <td>0.813299</td>\n",
              "      <td>101.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>761.0</td>\n",
              "      <td>0.685838</td>\n",
              "      <td>0.000901</td>\n",
              "      <td>75306.0</td>\n",
              "      <td>0.671892</td>\n",
              "      <td>0.469524</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.152464</td>\n",
              "      <td>95.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>15877.0</td>\n",
              "      <td>0.975536</td>\n",
              "      <td>6.144333e-05</td>\n",
              "      <td>9.946380e+06</td>\n",
              "      <td>0.982354</td>\n",
              "      <td>0.131662</td>\n",
              "      <td>116.0</td>\n",
              "      <td>0.834024</td>\n",
              "      <td>147.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <th>1</th>\n",
              "      <th>0</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>221.0</td>\n",
              "      <td>0.292312</td>\n",
              "      <td>0.001323</td>\n",
              "      <td>77528.0</td>\n",
              "      <td>0.240300</td>\n",
              "      <td>0.427265</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.324050</td>\n",
              "      <td>55.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>6697.0</td>\n",
              "      <td>0.521938</td>\n",
              "      <td>7.793613e-05</td>\n",
              "      <td>1.540540e+06</td>\n",
              "      <td>0.452550</td>\n",
              "      <td>0.497743</td>\n",
              "      <td>44.0</td>\n",
              "      <td>0.898566</td>\n",
              "      <td>62.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"5\" valign=\"top\">5</th>\n",
              "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
              "      <th>5</th>\n",
              "      <td>-1.285714</td>\n",
              "      <td>5025.0</td>\n",
              "      <td>-0.167506</td>\n",
              "      <td>-0.000033</td>\n",
              "      <td>1695870.0</td>\n",
              "      <td>0.072309</td>\n",
              "      <td>0.258999</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.069476</td>\n",
              "      <td>314.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0.571429</td>\n",
              "      <td>5025.0</td>\n",
              "      <td>-0.113199</td>\n",
              "      <td>-0.000023</td>\n",
              "      <td>1570250.0</td>\n",
              "      <td>0.023653</td>\n",
              "      <td>0.151964</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.062620</td>\n",
              "      <td>314.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0.510204</td>\n",
              "      <td>5025.0</td>\n",
              "      <td>0.242827</td>\n",
              "      <td>0.000048</td>\n",
              "      <td>1444630.0</td>\n",
              "      <td>0.069378</td>\n",
              "      <td>0.254097</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.074261</td>\n",
              "      <td>314.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1135001.0</td>\n",
              "      <td>0.740394</td>\n",
              "      <td>6.523285e-07</td>\n",
              "      <td>2.278223e+09</td>\n",
              "      <td>0.666720</td>\n",
              "      <td>0.471386</td>\n",
              "      <td>380.0</td>\n",
              "      <td>0.624708</td>\n",
              "      <td>454.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>-1.285714</td>\n",
              "      <td>5025.0</td>\n",
              "      <td>-0.425301</td>\n",
              "      <td>-0.000085</td>\n",
              "      <td>1193390.0</td>\n",
              "      <td>0.074148</td>\n",
              "      <td>0.262011</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.045234</td>\n",
              "      <td>314.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>5025.0</td>\n",
              "      <td>0.150114</td>\n",
              "      <td>0.000030</td>\n",
              "      <td>1067770.0</td>\n",
              "      <td>0.140606</td>\n",
              "      <td>0.347615</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.045610</td>\n",
              "      <td>314.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>422501.0</td>\n",
              "      <td>0.880197</td>\n",
              "      <td>2.083302e-06</td>\n",
              "      <td>6.142733e+08</td>\n",
              "      <td>0.866424</td>\n",
              "      <td>0.340197</td>\n",
              "      <td>134.0</td>\n",
              "      <td>0.897946</td>\n",
              "      <td>169.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>74 rows × 30 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         gradient_descent                                                \\\n",
              "      approximation ratio circuit count expectation approximation ratio   \n",
              "n p i                                                                     \n",
              "3 1 0            1.000000         237.0                        0.612424   \n",
              "    1           -0.333333         237.0                        0.000533   \n",
              "  2 0            1.000000         761.0                        0.618410   \n",
              "    1            1.000000         761.0                        0.685838   \n",
              "4 1 0            1.000000         221.0                        0.292312   \n",
              "...                   ...           ...                             ...   \n",
              "5 4 5           -1.285714        5025.0                       -0.167506   \n",
              "    6            0.571429        5025.0                       -0.113199   \n",
              "    7            0.510204        5025.0                        0.242827   \n",
              "    8           -1.285714        5025.0                       -0.425301   \n",
              "    9            1.000000        5025.0                        0.150114   \n",
              "\n",
              "                                                               \\\n",
              "      expectation approximation ratio per circuits gate count   \n",
              "n p i                                                           \n",
              "3 1 0                                     0.002584    34032.0   \n",
              "    1                                     0.000002    42540.0   \n",
              "  2 0                                     0.000813    61614.0   \n",
              "    1                                     0.000901    75306.0   \n",
              "4 1 0                                     0.001323    77528.0   \n",
              "...                                            ...        ...   \n",
              "5 4 5                                    -0.000033  1695870.0   \n",
              "    6                                    -0.000023  1570250.0   \n",
              "    7                                     0.000048  1444630.0   \n",
              "    8                                    -0.000085  1193390.0   \n",
              "    9                                     0.000030  1067770.0   \n",
              "\n",
              "                                                                           \\\n",
              "      groundspace overlap groundspace sharpness num_fun_calls path length   \n",
              "n p i                                                                       \n",
              "3 1 0            0.575877              0.494209           NaN    0.010990   \n",
              "    1            0.141675              0.348716           NaN    0.450001   \n",
              "  2 0            0.601489              0.489592           NaN    0.024554   \n",
              "    1            0.671892              0.469524           NaN    0.152464   \n",
              "4 1 0            0.240300              0.427265           NaN    0.324050   \n",
              "...                   ...                   ...           ...         ...   \n",
              "5 4 5            0.072309              0.258999           NaN    0.069476   \n",
              "    6            0.023653              0.151964           NaN    0.062620   \n",
              "    7            0.069378              0.254097           NaN    0.074261   \n",
              "    8            0.074148              0.262011           NaN    0.045234   \n",
              "    9            0.140606              0.347615           NaN    0.045610   \n",
              "\n",
              "              ...                tdvp                \\\n",
              "       steps  ... approximation ratio circuit count   \n",
              "n p i         ...                                     \n",
              "3 1 0   59.0  ...                 1.0        2773.0   \n",
              "    1   59.0  ...                 1.0        3133.0   \n",
              "  2 0   95.0  ...                 1.0       10909.0   \n",
              "    1   95.0  ...                 1.0       15877.0   \n",
              "4 1 0   55.0  ...                 1.0        6697.0   \n",
              "...      ...  ...                 ...           ...   \n",
              "5 4 5  314.0  ...                 NaN           NaN   \n",
              "    6  314.0  ...                 NaN           NaN   \n",
              "    7  314.0  ...                 1.0     1135001.0   \n",
              "    8  314.0  ...                 NaN           NaN   \n",
              "    9  314.0  ...                 1.0      422501.0   \n",
              "\n",
              "                                       \\\n",
              "      expectation approximation ratio   \n",
              "n p i                                   \n",
              "3 1 0                        0.858965   \n",
              "    1                        0.963212   \n",
              "  2 0                        0.885245   \n",
              "    1                        0.975536   \n",
              "4 1 0                        0.521938   \n",
              "...                               ...   \n",
              "5 4 5                             NaN   \n",
              "    6                             NaN   \n",
              "    7                        0.740394   \n",
              "    8                             NaN   \n",
              "    9                        0.880197   \n",
              "\n",
              "                                                                  \\\n",
              "      expectation approximation ratio per circuits    gate count   \n",
              "n p i                                                              \n",
              "3 1 0                                 3.097601e-04  6.080400e+05   \n",
              "    1                                 3.074409e-04  8.634480e+05   \n",
              "  2 0                                 8.114813e-05  5.531016e+06   \n",
              "    1                                 6.144333e-05  9.946380e+06   \n",
              "4 1 0                                 7.793613e-05  1.540540e+06   \n",
              "...                                            ...           ...   \n",
              "5 4 5                                          NaN           NaN   \n",
              "    6                                          NaN           NaN   \n",
              "    7                                 6.523285e-07  2.278223e+09   \n",
              "    8                                          NaN           NaN   \n",
              "    9                                 2.083302e-06  6.142733e+08   \n",
              "\n",
              "                                                                           \\\n",
              "      groundspace overlap groundspace sharpness num_fun_calls path length   \n",
              "n p i                                                                       \n",
              "3 1 0            0.766475              0.423073          56.0    0.938520   \n",
              "    1            0.971549              0.166258          62.0    0.886988   \n",
              "  2 0            0.881304              0.323430          74.0    0.813299   \n",
              "    1            0.982354              0.131662         116.0    0.834024   \n",
              "4 1 0            0.452550              0.497743          44.0    0.898566   \n",
              "...                   ...                   ...           ...         ...   \n",
              "5 4 5                 NaN                   NaN           0.0         NaN   \n",
              "    6                 NaN                   NaN           0.0         NaN   \n",
              "    7            0.666720              0.471386         380.0    0.624708   \n",
              "    8                 NaN                   NaN           0.0         NaN   \n",
              "    9            0.866424              0.340197         134.0    0.897946   \n",
              "\n",
              "              \n",
              "       steps  \n",
              "n p i         \n",
              "3 1 0   77.0  \n",
              "    1   87.0  \n",
              "  2 0  101.0  \n",
              "    1  147.0  \n",
              "4 1 0   62.0  \n",
              "...      ...  \n",
              "5 4 5    NaN  \n",
              "    6    NaN  \n",
              "    7  454.0  \n",
              "    8    NaN  \n",
              "    9  169.0  \n",
              "\n",
              "[74 rows x 30 columns]"
            ]
          },
          "execution_count": 342,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data_total.astype('float')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 337,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "execution_count": 337,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data_total['tdvp','path length'].dropna() "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 370,
      "metadata": {},
      "outputs": [],
      "source": [
        "res = statistics.ttest_ind(data_total['gd','path length'].astype('float'), data_total['tdvp','path length'].dropna().astype('float'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 349,
      "metadata": {},
      "outputs": [
        {
          "ename": "SyntaxError",
          "evalue": "incomplete input (1993234629.py, line 1)",
          "output_type": "error",
          "traceback": [
            "\u001b[0;36m  Cell \u001b[0;32mIn [349], line 1\u001b[0;36m\u001b[0m\n\u001b[0;31m    @1\u001b[0m\n\u001b[0m      ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m incomplete input\n"
          ]
        }
      ],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 355,
      "metadata": {},
      "outputs": [],
      "source": [
        "data_total.rename({'gradient_descent':'gd'}, axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 375,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>scipy</th>\n",
              "      <th>gd</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>approximation ratio</th>\n",
              "      <td>0.372538</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>circuit count</th>\n",
              "      <td>0.001083</td>\n",
              "      <td>0.001169</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>expectation approximation ratio</th>\n",
              "      <td>0.420896</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>expectation approximation ratio per circuits</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.075308</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gate count</th>\n",
              "      <td>0.00437</td>\n",
              "      <td>0.004409</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>groundspace overlap</th>\n",
              "      <td>0.090697</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>groundspace sharpness</th>\n",
              "      <td>0.245324</td>\n",
              "      <td>0.026734</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>path length</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 scipy        gd\n",
              "approximation ratio                           0.372538       0.0\n",
              "circuit count                                 0.001083  0.001169\n",
              "expectation approximation ratio               0.420896       0.0\n",
              "expectation approximation ratio per circuits       0.0  0.075308\n",
              "gate count                                     0.00437  0.004409\n",
              "groundspace overlap                           0.090697       0.0\n",
              "groundspace sharpness                         0.245324  0.026734\n",
              "path length                                        0.0       0.0"
            ]
          },
          "execution_count": 375,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "total_ttest = total_effect_size.copy()\n",
        "for meas in total_effect_size.index:\n",
        "    for alg in [\"scipy\", \"gd\"]:\n",
        "        total_ttest.loc[meas, alg] =  statistics.ttest_ind(\n",
        "            data_total['tdvp', meas].dropna().astype('float'),\n",
        "            data_total[alg, meas].astype('float'),\n",
        "        )[1]\n",
        "total_ttest"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "qutip-env",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6 | packaged by conda-forge | (main, Aug 22 2022, 20:35:26) [GCC 10.4.0]"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "fe45c953e7d91ced0867b19789acd0ab1d9439aa0b1f4fae31fd46e88f74aeab"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}