aggregate_results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "\n",
    "import os\n",
    "import argparse\n",
    "import functions\n",
    "import itertools\n",
    "\n",
    "from pathlib import Path\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def listdir_nohidden(path):\n",
    "    for f in os.listdir(path):\n",
    "        if not f.startswith('.'):\n",
    "            yield f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def number_to_str(n):\n",
    "    if n == 0:\n",
    "        return 0\n",
    "    return \"{0:.3e}\".format(n)\n",
    "#     if n > 1000:\n",
    "#         return \"{0:.4e}\".format(n)\n",
    "#     else:\n",
    "#         return \"{0:.2f}\".format(n)\n",
    "\n",
    "def print_table(results):\n",
    "    print(\"Function &Best &Worst &Median &$c$ &$\\\\bar\\{v\\}$ &Mean &Std & FRate\\\\\\\\\")\n",
    "    print(\"\\\\midrule\")\n",
    "    for k, v in results.items():\n",
    "        print(\"{0} &{1}({2}) &{3}({4}) &{5}({6}) &{7} &{8} &{9} &{10} &{11}\\\\\\\\\".format(k, \n",
    "                                                                                         number_to_str(v['best_perf']), \n",
    "                                                                                         v['best_consts'], \n",
    "                                                                                         number_to_str(v['worst_perf']), \n",
    "                                                                                         v['worst_consts'], \n",
    "                                                                                         number_to_str(v['median_perf']),\n",
    "                                                                                         v['median_consts'],\n",
    "                                                                                         tuple(v['c']),\n",
    "                                                                                         number_to_str(v['v']),\n",
    "                                                                                         number_to_str(v['mean']),\n",
    "                                                                                         number_to_str(v['std']),\n",
    "                                                                                         v['f_rate']\n",
    "                                                                                         ))\n",
    "\n",
    "def print_table_rev(results):\n",
    "    key_order = [(\"best_perf\", \"Best\"), (\"worst_perf\", \"Worst\"), (\"median_perf\", \"Median\"), (\"c\", \"$c$\"), (\"v\", \"$\\\\bar{v}$\"), (\"mean\", \"Mean\"), (\"std\", \"Std\"), (\"f_rate\", \"F Rate\")]\n",
    "    print(\"\\\\begin{tabular}{c|cccc}\")\n",
    "    print(\"\\\\toprule\")\n",
    "    print(\"&\" + \"&\".join(results.keys()) + \"\\\\\\\\\")\n",
    "    print(\"\\\\midrule\")\n",
    "    for k, n in key_order:\n",
    "        if k == \"best_perf\":\n",
    "            to_print = [f\"{h[0]}({h[1]})\" for h in list(zip([str(d[\"best_perf\"]) for d in results.values()], [str(d[\"best_consts\"]) for d in results.values()]))]\n",
    "            print(n + \"&\" + \"&\".join(to_print) + \"\\\\\\\\\")\n",
    "        elif k == \"worst_perf\":\n",
    "            to_print = [f\"{h[0]}({h[1]})\" for h in list(zip([str(d[\"worst_perf\"]) for d in results.values()], [str(d[\"worst_consts\"]) for d in results.values()]))]\n",
    "            print(n + \"&\" + \"&\".join(to_print) + \"\\\\\\\\\")\n",
    "        else:\n",
    "            print(n + \"&\" + \"&\".join([str(d[k]) for d in results.values()]) + \"\\\\\\\\\")\n",
    "    print(\"\\\\bottomrule\")\n",
    "    print(\"\\\\end{tabular}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate_results(_d, _r, _p):\n",
    "    dimensions = _d\n",
    "    runs = _r\n",
    "    logdir_path = _p\n",
    "    logdir = Path(logdir_path)\n",
    "    results = dict()\n",
    "    for c in sorted(listdir_nohidden(logdir)):\n",
    "        best_list = list()\n",
    "        idx_best_list = list()\n",
    "        performances = np.array([])\n",
    "\n",
    "        function_class = getattr(functions, c)\n",
    "        function_obj = function_class(dimensions=dimensions)\n",
    "\n",
    "        # feasibility rate\n",
    "        fes = 0\n",
    "        for r in listdir_nohidden(logdir / c):\n",
    "            if r == \"plots\":\n",
    "                continue\n",
    "            s = np.load(logdir / c/ r / \"solutions.npy\")\n",
    "            p = np.load(logdir / c / r / \"performances.npy\")\n",
    "            if len(performances) == 0:\n",
    "                performances = np.array([p])\n",
    "                solutions = np.array([s])\n",
    "            else:\n",
    "                performances = np.concatenate([performances, [p]], axis=0)\n",
    "                solutions = np.concatenate([solutions, [s]], axis=0)\n",
    "\n",
    "            # take best of this run\n",
    "            idx_best = np.unravel_index(np.nanargmin(p), p.shape)\n",
    "            idx_best_list.append(idx_best)\n",
    "            best_list.append(p[idx_best])\n",
    "\n",
    "            feasible_solution = False\n",
    "            # check if there is at least one feasible solution in this run to compute the feasible rate\n",
    "            # get all positions where there might be feasible solutions (0 for g and 0/1 for h constraints)\n",
    "            consts = list(function_class.constraints(None).keys())\n",
    "            combinations = list(map(list, itertools.product([0, 1], repeat=len(consts))))\n",
    "            for i, comb in enumerate(combinations):\n",
    "                for const, (k, item) in zip(consts, enumerate(comb)):\n",
    "                    if const.startswith('g'):\n",
    "                        combinations[i][k] = 0\n",
    "            # remove duplicates\n",
    "            combinations.sort()\n",
    "            feasible_indexes = list(l for l,_ in itertools.groupby(combinations))\n",
    "            for f_idx in feasible_indexes:\n",
    "                if p[tuple(f_idx)] != np.inf:\n",
    "                    feasible_solution = True\n",
    "            if feasible_solution:\n",
    "                fes += 1\n",
    "\n",
    "        performances[performances == np.inf] = np.nan\n",
    "\n",
    "        # get the indexes over the entire data structure, because we need to know which constraints were violated\n",
    "        best = np.min(best_list)\n",
    "        idx_best =  np.unravel_index(np.where(performances.flatten() == best)[0][0], performances.shape)\n",
    "        worst = np.max(best_list)\n",
    "        idx_worst = np.unravel_index(np.where(performances.flatten() == worst)[0][0], performances.shape)\n",
    "        idx_median = np.unravel_index(\n",
    "            np.where(performances.flatten() == np.nanpercentile(performances,50,interpolation='nearest'))[0][0], \n",
    "            performances.shape\n",
    "        )\n",
    "        median = performances[idx_median]\n",
    "\n",
    "        mean = np.mean(best_list)\n",
    "        std = np.std(best_list)\n",
    "\n",
    "        # compute # of violated constraints\n",
    "        consts_best = 0\n",
    "        consts_worst = 0\n",
    "        consts_median = 0\n",
    "        consts_median_specific = [0, 0, 0]\n",
    "        mean_violations = list()\n",
    "        for k, cost in enumerate(list(function_class.constraints(None).keys())):\n",
    "            if cost.startswith('g'):\n",
    "                if idx_best[k+1] > 0:\n",
    "                    consts_best += 1\n",
    "                if idx_worst[k+1] > 0:\n",
    "                    consts_worst += 1\n",
    "                if idx_median[k+1] > 0:\n",
    "                    consts_median += 1\n",
    "                    # add constraint violation to mean_violations\n",
    "                    mean_violations.append(function_obj.constraints()[cost]['func'](solutions[idx_median]))\n",
    "\n",
    "            if cost.startswith('h'):\n",
    "                if idx_best[k+1] > 1:\n",
    "                    consts_best += 1\n",
    "                if idx_worst[k+1] > 1:\n",
    "                    consts_worst += 1\n",
    "                if idx_median[k+1] > 1:\n",
    "                    consts_median += 1\n",
    "                    # add constraint violation to mean_violations\n",
    "                    mean_violations.append(function_obj.constraints()[cost]['func'](solutions[idx_median]))\n",
    "\n",
    "            # violation > 0.0001\n",
    "            if idx_median[k+1] == 2:\n",
    "                consts_median_specific[0] += 1\n",
    "            # violation > 0.01\n",
    "            if idx_median[k+1] == 3:\n",
    "                consts_median_specific[1] += 1\n",
    "            # violation > 1.0\n",
    "            if idx_median[k+1] == 4:\n",
    "                consts_median_specific[2] += 1\n",
    "\n",
    "    #     assert sum(consts_median_specific) == consts_median\n",
    "        results[c] = {\n",
    "            \"best_perf\": round(best, 2),\n",
    "            \"best_consts\": consts_best,\n",
    "            \"worst_perf\": round(worst, 2),\n",
    "            \"worst_consts\": consts_worst,\n",
    "            \"median_perf\": round(median, 2),\n",
    "            \"median_consts\": consts_median,\n",
    "            \"mean\": round(mean, 2),\n",
    "            \"std\": round(std, 2),\n",
    "            \"c\": consts_median_specific,\n",
    "            \"v\": round(np.mean(mean_violations), 2) if len(mean_violations) > 0 else 0,\n",
    "            \"f_rate\": round(fes / runs, 2)\n",
    "        }\n",
    "    print_table(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate_results_cec(_d, _r, _p):\n",
    "    dimensions = _d\n",
    "    runs = _r\n",
    "    logdir_path = _p\n",
    "    logdir = Path(logdir_path)\n",
    "    results = dict()\n",
    "    for c in sorted(listdir_nohidden(logdir)):\n",
    "        performances = np.array([])\n",
    "\n",
    "        function_class = getattr(functions, c)\n",
    "        function_obj = function_class(dimensions=dimensions)\n",
    "\n",
    "        # feasibility rate\n",
    "        fes = 0\n",
    "        best_for_each_run = list()\n",
    "        best_for_each_run_idxs = list()\n",
    "        for r in listdir_nohidden(logdir / c):\n",
    "            if r == \"plots\":\n",
    "                continue\n",
    "            s = np.load(logdir / c/ r / \"solutions.npy\")\n",
    "            p = np.load(logdir / c / r / \"performances.npy\")\n",
    "            if len(performances) == 0:\n",
    "                performances = np.array([p])\n",
    "                solutions = np.array([s])\n",
    "            else:\n",
    "                performances = np.concatenate([performances, [p]], axis=0)\n",
    "                solutions = np.concatenate([solutions, [s]], axis=0)\n",
    "\n",
    "\n",
    "            feasible_solution = False\n",
    "            # - Go over all feasible solutions of this run. In case there are feasible solutions, take the best one.\n",
    "            # - If there are feasible solutions also increase the feasiblity rate.\n",
    "\n",
    "            # Get all positions where there might be feasible solutions (0 for g and 0/1 for h constraints)\n",
    "            # We do this by creating all combinations of binary arrays, which indicate the positions of the feasible solutions\n",
    "            # This is because h constranints are considered feasible also if they are violated in the first bin, so\n",
    "            # the positions of g constraints must be 0 to be satisfied and the positions of h constraints can be either 0 or 1\n",
    "            consts = list(function_class.constraints(None).keys())\n",
    "            combinations = list(map(list, itertools.product([0, 1], repeat=len(consts))))\n",
    "            for i, comb in enumerate(combinations):\n",
    "                for const, (k, item) in zip(consts, enumerate(comb)):\n",
    "                    if const.startswith('g'):\n",
    "                        combinations[i][k] = 0\n",
    "            # remove duplicates\n",
    "            combinations.sort()\n",
    "            # best feasible solution for this run\n",
    "            best_sol = np.inf\n",
    "            best_sol_idx = None\n",
    "            feasible_indexes = list(l for l,_ in itertools.groupby(combinations))\n",
    "            for f_idx in feasible_indexes:\n",
    "                if p[tuple(f_idx)] != np.inf and p[tuple(f_idx)] < best_sol:\n",
    "                    best_sol = p[tuple(f_idx)]\n",
    "                    best_sol_idx = tuple(f_idx)\n",
    "                    feasible_solution = True\n",
    "            # there was at least one feasible solution\n",
    "            if feasible_solution:\n",
    "                fes += 1\n",
    "            # search for the best UNfeasible solution\n",
    "            else:            \n",
    "                # if there is not feasible solution, then we can take the best unfeasible one. \n",
    "                # The unfeasible solutions are ordered by the mean value of the violations of all constraints\n",
    "                # Basically the best unfeasible solution is the one with best v\n",
    "                # best_sol_idx = np.unravel_index(np.nanargmin(p), p.shape)\n",
    "                # best_sol = p[best_sol_idx]\n",
    "                # compute `v` for every element in the map of elites (they are all unfeasible solutions because there are no feasible ones)\n",
    "                best_unfeasible = np.inf\n",
    "                best_unfeasible_idx = None\n",
    "                best_unfeasible_v = np.inf\n",
    "                for idx, value in np.ndenumerate(p):\n",
    "                    if value == np.inf:\n",
    "                        continue\n",
    "                    current_v = list()\n",
    "                    for k, cost in enumerate(list(function_class.constraints(None).keys())):\n",
    "                        if cost.startswith('g') and idx[k] > 0:\n",
    "                            current_v.append(function_obj.constraints()[cost]['func'](s[idx]))\n",
    "                        if cost.startswith('h') and idx[k] > 1:\n",
    "                            current_v.append(function_obj.constraints()[cost]['func'](s[idx]))\n",
    "                    if len(current_v) == 0:\n",
    "                        raise ValueError(\"MeanViolations were not found, but there are no feasible solutions!\")\n",
    "                    current_v = np.mean(current_v)\n",
    "                    if current_v < best_unfeasible_v:\n",
    "                        best_unfeasible = value\n",
    "                        best_unfeasible_idx = idx\n",
    "                        best_unfeasible_v = current_v\n",
    "                best_sol = best_unfeasible\n",
    "                best_sol_idx = best_unfeasible_idx\n",
    "\n",
    "            best_for_each_run.append(best_sol)\n",
    "            best_for_each_run_idxs.append(best_sol_idx)\n",
    "\n",
    "\n",
    "        # ------------------------------------------------------------------------------\n",
    "        # So now should have the list of best solutions and the correct feasibility rate\n",
    "        # ------------------------------------------------------------------------------\n",
    "\n",
    "        performances[performances == np.inf] = np.nan\n",
    "\n",
    "        # get the indexes over the entire data structure, because we need to know which constraints were violated\n",
    "        best = np.min(best_for_each_run)\n",
    "        idx_best =  np.unravel_index(np.where(performances.flatten() == best)[0][0], performances.shape)\n",
    "        worst = np.max(best_for_each_run)\n",
    "        idx_worst = np.unravel_index(np.where(performances.flatten() == worst)[0][0], performances.shape)\n",
    "\n",
    "        idx_median = np.where(np.array(best_for_each_run) == np.nanpercentile(best_for_each_run,50,interpolation='nearest'))[0][0]\n",
    "        median = best_for_each_run[idx_median]\n",
    "        # now get the index in the performances structure\n",
    "        idx_median = np.unravel_index(np.where(performances.flatten() == median)[0][0], performances.shape)\n",
    "\n",
    "        mean = np.mean(best_for_each_run)\n",
    "        std = np.std(best_for_each_run)\n",
    "\n",
    "        # compute # of violated constraints\n",
    "        consts_best = 0\n",
    "        consts_worst = 0\n",
    "        consts_median = 0\n",
    "        consts_median_specific = [0, 0, 0]\n",
    "        mean_violations = list()\n",
    "        for k, cost in enumerate(list(function_class.constraints(None).keys())):\n",
    "            if cost.startswith('g'):\n",
    "                if idx_best[k+1] > 0:\n",
    "                    consts_best += 1\n",
    "                if idx_worst[k+1] > 0:\n",
    "                    consts_worst += 1\n",
    "                if idx_median[k+1] > 0:\n",
    "                    consts_median += 1\n",
    "                    # add constraint violation to mean_violations\n",
    "                    mean_violations.append(function_obj.constraints()[cost]['func'](solutions[idx_median]))\n",
    "\n",
    "            if cost.startswith('h'):\n",
    "                if idx_best[k+1] > 1:\n",
    "                    consts_best += 1\n",
    "                if idx_worst[k+1] > 1:\n",
    "                    consts_worst += 1\n",
    "                if idx_median[k+1] > 1:\n",
    "                    consts_median += 1\n",
    "                    # add constraint violation to mean_violations\n",
    "                    mean_violations.append(function_obj.constraints()[cost]['func'](solutions[idx_median]))\n",
    "\n",
    "            # violation > 0.0001\n",
    "            if idx_median[k+1] == 2:\n",
    "                consts_median_specific[0] += 1\n",
    "            # violation > 0.01\n",
    "            if idx_median[k+1] == 3:\n",
    "                consts_median_specific[1] += 1\n",
    "            # violation > 1.0\n",
    "            if idx_median[k+1] == 4:\n",
    "                consts_median_specific[2] += 1\n",
    "\n",
    "    #     assert sum(consts_median_specific) == consts_median\n",
    "        results[c] = {\n",
    "            \"best_perf\": round(best, 2),\n",
    "            \"best_consts\": consts_best,\n",
    "            \"worst_perf\": round(worst, 2),\n",
    "            \"worst_consts\": consts_worst,\n",
    "            \"median_perf\": round(median, 2),\n",
    "            \"median_consts\": consts_median,\n",
    "            \"mean\": round(mean, 2),\n",
    "            \"std\": round(std, 2),\n",
    "            \"c\": consts_median_specific,\n",
    "            \"v\": round(np.mean(mean_violations), 2) if len(mean_violations) > 0 else 0,\n",
    "            \"f_rate\": round(fes / runs, 2)\n",
    "        }\n",
    "    print_table(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dimensions = 10\n",
    "runs = 25\n",
    "logdir_path = 'log/complete_logs/run_10D_crossover/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Function &Best &Worst &Median &$c$ &$\\bar\\{v\\}$ &Mean &Std & FRate\\\\\n",
      "\\midrule\n",
      "C01 &-1.310e+00(1) &-4.600e-01(1) &-1.600e-01(1) &(0, 0, 1) &8.000e-01 &-7.300e-01 &2.000e-01 &1.0\\\\\n",
      "C02 &-3.390e+00(2) &-2.580e+00(2) &-9.600e-01(2) &(0, 1, 1) &-6.880e+00 &-3.050e+00 &2.200e-01 &0.44\\\\\n",
      "C03 &1.154e+14(1) &9.250e+14(1) &4.362e+14(1) &(0, 0, 1) &1.310e+06 &5.005e+14 &2.213e+14 &0.0\\\\\n",
      "C04 &-4.441e+01(4) &-6.460e+00(4) &9.600e-01(4) &(0, 1, 3) &5.664e+02 &-3.514e+01 &1.417e+01 &0.0\\\\\n",
      "C05 &-5.348e+02(2) &-7.938e+01(2) &-1.681e+02(2) &(0, 0, 2) &-1.972e+01 &-3.187e+02 &1.544e+02 &0.0\\\\\n",
      "C06 &-4.336e+02(2) &-1.328e+01(2) &-9.162e+01(2) &(0, 1, 1) &8.514e+01 &-2.528e+02 &1.276e+02 &0.0\\\\\n",
      "C07 &6.152e+03(0) &5.908e+07(0) &4.109e+09(1) &(0, 1, 0) &-7.000e-01 &4.070e+06 &1.251e+07 &1.0\\\\\n",
      "C08 &2.242e+03(1) &1.388e+06(1) &4.268e+05(0) &(0, 0, 0) &0 &3.925e+05 &4.921e+05 &1.0\\\\\n",
      "C09 &9.236e+04(1) &2.809e+13(1) &1.784e+07(0) &(0, 0, 0) &0 &2.474e+12 &7.129e+12 &0.2\\\\\n",
      "C10 &2.309e+03(1) &3.539e+13(1) &4.490e+06(1) &(1, 0, 0) &7.495e+01 &1.715e+12 &7.029e+12 &0.44\\\\\n",
      "C11 &-5.242e+01(1) &-3.835e+01(1) &-4.532e+01(1) &(0, 0, 1) &3.883e+12 &-4.474e+01 &3.120e+00 &0.0\\\\\n",
      "C12 &-7.742e+03(2) &-5.137e+03(1) &-5.810e+03(2) &(0, 0, 2) &1.590e+12 &-6.393e+03 &7.088e+02 &0.0\\\\\n",
      "C13 &-4.530e+02(2) &-3.103e+02(2) &-3.952e+02(2) &(0, 1, 1) &8.816e+02 &-3.981e+02 &3.381e+01 &0.0\\\\\n",
      "C14 &6.917e+04(1) &1.500e+13(1) &1.815e+09(2) &(1, 0, 1) &-3.715e+02 &9.907e+11 &3.413e+12 &1.0\\\\\n",
      "C15 &3.170e+03(1) &2.775e+11(2) &1.046e+08(1) &(0, 1, 0) &4.132e+02 &1.235e+10 &5.425e+10 &1.0\\\\\n",
      "C16 &4.000e-02(3) &1.500e-01(4) &2.400e-01(3) &(0, 1, 2) &1.421e+03 &8.000e-02 &2.000e-02 &0.12\\\\\n",
      "C17 &2.100e-01(2) &1.460e+00(3) &3.320e+00(2) &(1, 1, 0) &2.200e+00 &6.600e-01 &2.400e-01 &0.52\\\\\n",
      "C18 &5.000e-01(2) &2.305e+01(1) &1.260e+01(1) &(0, 0, 1) &-3.610e+00 &6.750e+00 &6.360e+00 &0.92\\\\\n"
     ]
    }
   ],
   "source": [
    "aggregate_results(dimensions, runs, logdir_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Function &Best &Worst &Median &$c$ &$\\bar\\{v\\}$ &Mean &Std & FRate\\\\\n",
      "\\midrule\n",
      "C01 &-8.300e-01(0) &-4.500e-01(0) &-5.500e-01(0) &(0, 0, 0) &0 &-5.700e-01 &1.000e-01 &1.0\\\\\n",
      "C02 &-2.140e+00(1) &4.920e+00(0) &2.870e+00(2) &(1, 1, 0) &-8.750e+00 &1.850e+00 &2.230e+00 &0.44\\\\\n",
      "C03 &1.154e+14(1) &9.250e+14(1) &4.362e+14(1) &(0, 0, 1) &1.310e+06 &5.005e+14 &2.213e+14 &0.0\\\\\n",
      "C04 &-4.441e+01(4) &5.003e+01(4) &3.800e-01(4) &(0, 2, 2) &9.570e+01 &1.270e+00 &2.920e+01 &0.0\\\\\n",
      "C05 &-4.801e+02(2) &1.527e+02(1) &-2.316e+02(2) &(0, 1, 1) &1.188e+01 &-2.181e+02 &1.804e+02 &0.0\\\\\n",
      "C06 &-4.275e+02(2) &4.210e+02(2) &-1.927e+02(1) &(0, 0, 1) &-1.397e+02 &-1.363e+02 &2.086e+02 &0.0\\\\\n",
      "C07 &6.152e+03(0) &5.908e+07(0) &3.171e+05(0) &(0, 0, 0) &0 &4.070e+06 &1.251e+07 &1.0\\\\\n",
      "C08 &4.550e+03(0) &1.391e+06(0) &1.518e+05(0) &(0, 0, 0) &0 &4.183e+05 &4.949e+05 &1.0\\\\\n",
      "C09 &8.301e+05(1) &1.600e+15(1) &8.227e+07(1) &(0, 1, 0) &-6.704e+01 &1.612e+14 &4.344e+14 &0.2\\\\\n",
      "C10 &2.482e+04(1) &7.494e+14(1) &7.169e+07(1) &(1, 0, 0) &-3.527e+01 &3.833e+13 &1.490e+14 &0.44\\\\\n",
      "C11 &-5.242e+01(1) &-3.835e+01(1) &-4.532e+01(1) &(0, 0, 1) &3.883e+12 &-4.474e+01 &3.120e+00 &0.0\\\\\n",
      "C12 &-7.742e+03(2) &3.236e+02(2) &-5.923e+03(2) &(1, 0, 1) &1.573e+12 &-5.795e+03 &1.603e+03 &0.0\\\\\n",
      "C13 &-4.474e+02(2) &-2.925e+02(2) &-3.952e+02(2) &(0, 1, 1) &8.816e+02 &-3.900e+02 &3.742e+01 &0.0\\\\\n",
      "C14 &1.033e+05(0) &3.280e+14(0) &1.724e+09(0) &(0, 0, 0) &0 &3.958e+13 &8.132e+13 &1.0\\\\\n",
      "C15 &4.980e+03(0) &4.089e+11(0) &9.060e+07(0) &(0, 0, 0) &0 &1.863e+10 &8.010e+10 &1.0\\\\\n",
      "C16 &1.200e-01(1) &1.100e+00(4) &2.900e-01(3) &(2, 1, 0) &-3.723e+05 &5.100e-01 &3.800e-01 &0.12\\\\\n",
      "C17 &7.300e-01(3) &5.915e+02(0) &1.123e+01(3) &(2, 1, 0) &-8.454e+05 &6.054e+01 &1.196e+02 &0.52\\\\\n",
      "C18 &2.430e+00(1) &4.493e+02(0) &5.091e+01(0) &(0, 0, 0) &0 &1.078e+02 &1.165e+02 &0.92\\\\\n"
     ]
    }
   ],
   "source": [
    "aggregate_results_cec(dimensions, runs, logdir_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}