Merge pull request #327 from dyson-ai/feature/video_grid

Feature/video grid

bencher/bench_cfg.py

@@ -303,6 +303,11 @@ class BenchCfg(BenchRunCfg):
         doc="store the hash value of the config to avoid having to hash multiple times",
     )
 
+    plot_callbacks = param.List(
+        None,
+        doc="A callable that takes a BenchResult and returns panel representation of the results",
+    )
+
     def __init__(self, **params):
         super().__init__(**params)
         self.plot_lib = None

bencher/bench_report.py

@@ -48,7 +48,7 @@ def append_col(self, pane: pn.panel, name: str = None) -> None:
         self.pane.append(col)
 
     def append_result(self, bench_res: BenchResult) -> None:
-        self.append_tab(bench_res.to_auto_plots(), bench_res.bench_cfg.title)
+        self.append_tab(bench_res.plot(), bench_res.bench_cfg.title)
 
     def append_tab(self, pane: pn.panel, name: str = None) -> None:
         if pane is not None:

bencher/bencher.py

@@ -10,6 +10,7 @@
 from diskcache import Cache
 from contextlib import suppress
 from functools import partial
+import panel as pn
 
 from bencher.worker_job import WorkerJob
 
@@ -171,6 +172,10 @@ def __init__(
         self.input_vars = None
         self.result_vars = None
         self.const_vars = None
+        self.plot_callbacks = []
+
+    def add_plot_callback(self, callback: Callable[[BenchResult], pn.panel]) -> None:
+        self.plot_callbacks.append(callback)
 
     def set_worker(self, worker: Callable, worker_input_cfg: ParametrizedSweep = None) -> None:
         """Set the benchmark worker function and optionally the type the worker expects
@@ -243,6 +248,7 @@ def plot_sweep(
         tag: str = "",
         run_cfg: BenchRunCfg = None,
         plot: bool = True,
+        plot_callbacks=None,
     ) -> BenchResult:
         """The all in 1 function benchmarker and results plotter.
 
@@ -257,7 +263,8 @@ def plot_sweep(
             pass_repeat (bool,optional) By default do not pass the kwarg 'repeat' to the benchmark function.  Set to true if
             you want the benchmark function to be passed the repeat number
             tag (str,optional): Use tags to group different benchmarks together.
-            run_cfg: (BenchRunCfg, optional): A config for storing how the benchmarks and run and plotted
+            run_cfg: (BenchRunCfg, optional): A config for storing how the benchmarks and run
+            plot_callbacks: A list of plot callbacks to clal on the results
         Raises:
             ValueError: If a result variable is not set
 
@@ -369,6 +376,12 @@ def plot_sweep(
                 "## Results Description\nPlease set post_description to explain these results"
             )
 
+        if plot_callbacks is None:
+            if len(self.plot_callbacks) == 0:
+                plot_callbacks = [BenchResult.to_auto_plots]
+            else:
+                plot_callbacks = self.plot_callbacks
+
         bench_cfg = BenchCfg(
             input_vars=input_vars,
             result_vars=result_vars_only,
@@ -381,6 +394,7 @@ def plot_sweep(
             pass_repeat=pass_repeat,
             tag=run_cfg.run_tag + tag,
             auto_plot=plot,
+            plot_callbacks=plot_callbacks,
         )
         return self.run_sweep(bench_cfg, run_cfg, time_src)
 
@@ -443,6 +457,7 @@ def run_sweep(
 
         if bench_cfg.auto_plot:
             self.report.append_result(bench_res)
+
         self.results.append(bench_res)
         return bench_res
 

bencher/example/example_image.py

@@ -44,7 +44,7 @@ def points_to_polygon_png(self, points: list[float], filename: str):
 
         ax.set_aspect("equal")
         fig.add_axes(ax)
-        fig.savefig(filename, dpi=50)
+        fig.savefig(filename, dpi=30)
 
         return filename
 
@@ -54,24 +54,9 @@ def example_image(
 ) -> bch.Bench:
     bench = bch.Bench("polygons", BenchPolygons(), run_cfg=run_cfg, report=report)
 
-    for s in [
-        [BenchPolygons.param.sides],
-        [BenchPolygons.param.sides, BenchPolygons.param.linewidth],
-        [BenchPolygons.param.sides, BenchPolygons.param.linewidth, BenchPolygons.param.linestyle],
-        [
-            BenchPolygons.param.sides,
-            BenchPolygons.param.linewidth,
-            BenchPolygons.param.linestyle,
-            BenchPolygons.param.color,
-        ],
-        [
-            BenchPolygons.param.sides,
-            BenchPolygons.param.linewidth,
-            BenchPolygons.param.linestyle,
-            BenchPolygons.param.color,
-            BenchPolygons.param.radius,
-        ],
-    ]:
+    sweep_vars = ["sides", "radius", "linewidth", "color"]
+    for i in range(1, len(sweep_vars)):
+        s = sweep_vars[:i]
         bench.plot_sweep(
             f"Polygons Sweeping {len(s)} Parameters",
             input_vars=s,
@@ -81,24 +66,26 @@ def example_image(
     return bench
 
 
-if __name__ == "__main__":
+def example_image_vid(
+    run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
+) -> bch.Bench:
+    bench = BenchPolygons().to_bench(run_cfg, report)
+    bench.add_plot_callback(bch.BenchResult.to_sweep_summary)
+    bench.add_plot_callback(bch.BenchResult.to_video_grid)
+    bench.plot_sweep(input_vars=["sides"])
+    bench.plot_sweep(input_vars=["sides", "radius"])
+    return bench
 
-    def example_image_vid(
-        run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
-    ) -> bch.Bench:
-        bench = BenchPolygons().to_bench(run_cfg, report)
-        bench.plot_sweep(input_vars=["sides", "radius", "color"], plot=True)
-        return bench
+
+if __name__ == "__main__":
 
     def example_image_vid_sequential(
         run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
     ) -> bch.Bench:
         bench = BenchPolygons().to_bench(run_cfg, report)
-        res_list = bench.sweep_sequential(
-            input_vars=["radius", "sides", "linewidth", "color"], group_size=3
-        )
-        for r in res_list:
-            bench.report.append(r.to_video_summary())
+        bench.add_plot_callback(bch.BenchResult.to_title)
+        bench.add_plot_callback(bch.BenchResult.to_video_grid)
+        bench.sweep_sequential(input_vars=["radius", "sides", "linewidth", "color"], group_size=1)
         return bench
 
     # def example_image_pairs()

bencher/results/bench_result.py

@@ -35,6 +35,14 @@ def default_plot_callbacks():
     def plotly_callbacks():
         return [HoloviewResult.to_surface, PlotlyResult.to_volume]
 
+    def plot(self) -> pn.panel:
+        """Plots the benchresult using the plot callbacks defined by the bench run
+
+        Returns:
+             pn.panel: A panel representation of the results
+        """
+        return pn.Column(*[cb(self) for cb in self.bench_cfg.plot_callbacks])
+
     def to_auto(
         self,
         plot_list: List[callable] = None,
@@ -67,7 +75,7 @@ def to_auto(
             )
         return row.pane
 
-    def to_auto_plots(self, **kwargs) -> List[pn.panel]:
+    def to_auto_plots(self, **kwargs) -> pn.panel:
         """Given the dataset result of a benchmark run, automatically dedeuce how to plot the data based on the types of variables that were sampled
 
         Args:

bencher/results/bench_result_base.py

@@ -5,17 +5,22 @@
 from param import Parameter
 import holoviews as hv
 from functools import partial
+import panel as pn
+
 from bencher.utils import int_to_col, color_tuple_to_css, callable_name
 
 from bencher.variables.parametrised_sweep import ParametrizedSweep
 from bencher.variables.results import OptDir
 from copy import deepcopy
 from bencher.results.optuna_result import OptunaResult
 from bencher.variables.results import ResultVar
-from bencher.results.float_formatter import FormatFloat
 from bencher.plotting.plot_filter import VarRange, PlotFilter
-import panel as pn
 
+from bencher.variables.results import (
+    ResultReference,
+)
+
+from bencher.results.composable_container.composable_container_panel import ComposableContainerPanel
 
 # todo add plugins
 # https://gist.github.com/dorneanu/cce1cd6711969d581873a88e0257e312
@@ -333,29 +338,24 @@ def _to_panes_da(
             time_dim_delta = 0
 
         if num_dims > (target_dimension + time_dim_delta) and num_dims != 0:
-            dim_sel = dims[-1]
+            selected_dim = dims[-1]
             # print(f"selected dim {dim_sel}")
-
             dim_color = color_tuple_to_css(int_to_col(num_dims - 2, 0.05, 1.0))
 
-            background_col = dim_color
-            name = " vs ".join(dims)
-
-            container_args = {"name": name, "styles": {"background": background_col}}
-            outer_container = (
-                pn.Row(**container_args) if horizontal else pn.Column(**container_args)
+            outer_container = ComposableContainerPanel(
+                name=" vs ".join(dims), background_col=dim_color, horizontal=not horizontal
             )
-
             max_len = 0
-
-            for i in range(dataset.sizes[dim_sel]):
-                sliced = dataset.isel({dim_sel: i})
-
-                lable_val = sliced.coords[dim_sel].values.item()
-                if isinstance(lable_val, (int, float)):
-                    lable_val = FormatFloat()(lable_val)
-
-                label = f"{dim_sel}={lable_val}"
+            for i in range(dataset.sizes[selected_dim]):
+                sliced = dataset.isel({selected_dim: i})
+                label_val = sliced.coords[selected_dim].values.item()
+                inner_container = ComposableContainerPanel(
+                    outer_container.name,
+                    width=num_dims - target_dimension,
+                    var_name=selected_dim,
+                    var_value=label_val,
+                    horizontal=horizontal,
+                )
 
                 panes = self._to_panes_da(
                     sliced,
@@ -364,35 +364,46 @@ def _to_panes_da(
                     horizontal=len(sliced.sizes) <= target_dimension + 1,
                     result_var=result_var,
                 )
-                width = num_dims - target_dimension
-
-                container_args = {
-                    "name": name,
-                    "styles": {"border-bottom": f"{width}px solid grey"},
-                }
-
-                if horizontal:
-                    inner_container = pn.Column(**container_args)
-                    align = ("center", "center")
-                else:
-                    inner_container = pn.Row(**container_args)
-                    align = ("end", "center")
-
-                label_len = len(label)
-                if label_len > max_len:
-                    max_len = label_len
-                side = pn.pane.Markdown(label, align=align)
-
-                inner_container.append(side)
+
+                if inner_container.label_len > max_len:
+                    max_len = inner_container.label_len
                 inner_container.append(panes)
-                outer_container.append(inner_container)
-                # outer_container.append(pn.Row(inner_container, align="center"))
-            for c in outer_container:
+                outer_container.append(inner_container.container)
+            for c in outer_container.container:
                 c[0].width = max_len * 7
         else:
             return plot_callback(dataset=dataset, result_var=result_var, **kwargs)
 
-        return outer_container
+        return outer_container.container
+
+    def zero_dim_da_to_val(self, da_ds: xr.DataArray | xr.Dataset) -> Any:
+        # todo this is really horrible, need to improve
+        dim = None
+        if isinstance(da_ds, xr.Dataset):
+            dim = list(da_ds.keys())[0]
+            da = da_ds[dim]
+        else:
+            da = da_ds
+
+        for k in da.coords.keys():
+            dim = k
+            break
+        if dim is None:
+            return da_ds.values.squeeze().item()
+        return da.expand_dims(dim).values[0]
+
+    def ds_to_container(
+        self, dataset: xr.Dataset, result_var: Parameter, container, **kwargs
+    ) -> Any:
+        val = self.zero_dim_da_to_val(dataset[result_var.name])
+        if isinstance(result_var, ResultReference):
+            ref = self.object_index[val]
+            val = ref.obj
+            if ref.container is not None:
+                return ref.container(val, **kwargs)
+        if container is not None:
+            return container(val, styles={"background": "white"}, **kwargs)
+        return val
 
     # MAPPING TO LOWER LEVEL BENCHCFG functions so they are available at a top level.
     def to_sweep_summary(self, **kwargs):

bencher/results/composable_container/composable_container_base.py

@@ -0,0 +1,58 @@
+from enum import Enum, auto
+from typing import Any
+from bencher.results.float_formatter import FormatFloat
+
+
+# TODO enable these options
+class ComposeType(Enum):
+    right = auto()  # append the container to the right (creates a row)
+    down = auto()  # append the container below (creates a column)
+    overlay = auto()  # overlay on top of the current container (alpha blending)
+    sequence = auto()  # display the container after (in time)
+
+
+class ComposableContainerBase:
+    """A base class for renderer backends.  A composable renderr"""
+
+    @staticmethod
+    def label_formatter(var_name: str, var_value: int | float | str) -> str:
+        """Take a variable name and values and return a pretty version with approximate fixed width
+
+        Args:
+            var_name (str): The name of the variable, usually a dimension
+            var_value (int | float | str): The value of the dimension
+
+        Returns:
+            str: Pretty string representation with fixed width
+        """
+
+        if isinstance(var_value, (int, float)):
+            var_value = FormatFloat()(var_value)
+        if var_name is not None and var_value is not None:
+            return f"{var_name}={var_value}"
+        if var_name is not None:
+            return f"{var_name}"
+        if var_value is not None:
+            return f"{var_value}"
+        return None
+
+    def __init__(self, horizontal: bool = True) -> None:
+        self.horizontal: bool = horizontal
+        self.compose_method = ComposeType.right
+        self.container = []
+
+    def append(self, obj: Any) -> None:
+        """Add an object to the container.  The relationship between the objects is defined by the ComposeType
+
+        Args:
+            obj (Any): Object to add to the container
+        """
+        self.container.append(obj)
+
+    def render(self):
+        """Return a representation of the container that can be composed with other render() results. This function can also be used to defer layout and rending options until all the information about the container content is known.  You may need to ovverride this method depending on the container. See composable_container_video as an example.
+
+        Returns:
+            Any: Visual representation of the container that can be combined with other containers
+        """
+        return self.container