Merge pull request #299 from dyson-ai/feature/fix_graph_units

Feature/fix graph units

bencher/example/example_simple_float.py

@@ -26,6 +26,7 @@ def example_1D_float(
         result_vars=[ExampleBenchCfgOut.param.out_sin],
         description=example_1D_float.__doc__,
     )
+
     return bench
 
 

bencher/results/bench_result_base.py

@@ -4,7 +4,6 @@
 import xarray as xr
 from param import Parameter
 import holoviews as hv
-import numpy as np
 from functools import partial
 from bencher.utils import int_to_col, color_tuple_to_css
 
@@ -58,38 +57,36 @@ def to_hv_dataset(
         Returns:
             hv.Dataset: results in the form of a holoviews dataset
         """
+        return hv.Dataset(self.to_dataset(reduce, result_var))
 
+    def to_dataset(
+        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None
+    ) -> xr.Dataset:
+        """Generate a summarised xarray dataset.
+
+        Args:
+            reduce (ReduceType, optional): Optionally perform reduce options on the dataset.  By default the returned dataset will calculate the mean and standard devation over the "repeat" dimension so that the dataset plays nicely with most of the holoviews plot types.  Reduce.Sqeeze is used if there is only 1 repeat and you want the "reduce" variable removed from the dataset. ReduceType.None returns an unaltered dataset. Defaults to ReduceType.AUTO.
+
+        Returns:
+            xr.Dataset: results in the form of an xarray dataset
+        """
         if reduce == ReduceType.AUTO:
             reduce = ReduceType.REDUCE if self.bench_cfg.repeats > 1 else ReduceType.SQUEEZE
 
-        vdims = [r.name for r in self.bench_cfg.result_vars]
-        kdims = [i.name for i in self.bench_cfg.all_vars]
-
         ds = self.ds if result_var is None else self.ds[result_var.name]
+
         match (reduce):
             case ReduceType.REDUCE:
-                # if result_var
-                vdims = []
-                non_sum = []
-                for r in self.bench_cfg.result_vars:
-                    if isinstance(r, ResultVar):
-                        vdims.append(r.name)
-                    else:
-                        non_sum.append(r.name)
-
-                ds_num = ds.drop_vars(non_sum)
-                return hv.Dataset(ds_num, kdims=kdims, vdims=vdims).reduce(
-                    ["repeat"], np.mean, np.std
-                )
+                ds_reduce_mean = ds.mean(dim="repeat", keep_attrs=True)
+                ds_reduce_std = ds.std(dim="repeat", keep_attrs=True)
+
+                for v in ds_reduce_mean.data_vars:
+                    ds_reduce_mean[f"{v}_std"] = ds_reduce_std[v]
+                return ds_reduce_mean
             case ReduceType.SQUEEZE:
-                return hv.Dataset(ds.squeeze(drop=True), vdims=vdims)
+                return ds.squeeze(drop=True)
             case _:
-                return hv.Dataset(ds, kdims=kdims, vdims=vdims)
-
-    def to_dataset(
-        self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None
-    ) -> xr.Dataset:
-        return self.to_hv_dataset(reduce=reduce, result_var=result_var).data
+                return ds
 
     def get_optimal_vec(
         self,

bencher/variables/parametrised_sweep.py

@@ -206,3 +206,8 @@ def gen_video_path(self, video_name: str) -> str:
 
     def gen_image_path(self, image_name: str, filetype=".png") -> str:
         return self.gen_path(image_name, "img", filetype)
+
+    def to_bench(self, run_cfg=None, report=None):
+        from bencher import Bench
+
+        return Bench(self.name, self, run_cfg=run_cfg, report=report)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "holobench"
-version = "1.2.0"
+version = "1.2.1"
 
 authors = [{ name = "Austin Gregg-Smith", email = "blooop@gmail.com" }]
 description = "A package for benchmarking the performance of arbitrary functions"

test/test_bench_result_base.py

@@ -0,0 +1,41 @@
+import unittest
+import bencher as bch
+
+from bencher.example.example_meta import BenchableObject
+
+
+class TestBenchResultBase(unittest.TestCase):
+    def test_to_dataset(self):
+        bench = BenchableObject().to_bench()
+
+        res_repeat1 = bench.plot_sweep(
+            "sweep1repeat",
+            input_vars=[BenchableObject.param.float1],
+            result_vars=[BenchableObject.param.distance, BenchableObject.param.sample_noise],
+            run_cfg=bch.BenchRunCfg(repeats=1),
+            plot=False,
+        )
+
+        res_repeat2 = bench.plot_sweep(
+            "sweep2repeat",
+            input_vars=[BenchableObject.param.float1],
+            result_vars=[BenchableObject.param.distance, BenchableObject.param.sample_noise],
+            run_cfg=bch.BenchRunCfg(repeats=2),
+            plot=False,
+        )
+
+        # print(res_repeat1.to_dataset())
+        # print(res_repeat1.to_hv_dataset().data)
+        # print(res_repeat1.to_hv_dataset_old().data)
+
+        # print(res_repeat1.to_dataset()["distance"].attrs)
+
+        # print(res_repeat2.to_dataset())
+        # print(res_repeat2.to_hv_dataset())
+        # print(res_repeat2.to_dataset()["distance"].attrs)
+
+        self.assertEqual(
+            res_repeat1.to_dataset()["distance"].attrs, res_repeat2.to_dataset()["distance"].attrs
+        )
+
+        # bm.__call__(float_vars=1, sample_with_repeats=1)

test/test_meta_tests.py

@@ -0,0 +1,55 @@
+import unittest
+import bencher as bch
+from bencher.example.example_meta import BenchableObject
+
+
+class TestBenchMeta(unittest.TestCase):
+    def test_repeats_equal(self):
+        # bench = bch.Bench("meta", BenchMeta())
+
+        bench = bch.Bench("bench", BenchableObject())
+
+        res1 = bench.plot_sweep(
+            "repeats", input_vars=[BenchableObject.param.float1], run_cfg=bch.BenchRunCfg(repeats=1)
+        )
+
+        res1_eq = bench.plot_sweep(
+            "repeats", input_vars=[BenchableObject.param.float1], run_cfg=bch.BenchRunCfg(repeats=1)
+        )
+
+        res2 = bench.plot_sweep(
+            "repeats", input_vars=[BenchableObject.param.float1], run_cfg=bch.BenchRunCfg(repeats=2)
+        )
+
+        self.assertTrue(
+            res1.to_dataset(bch.ReduceType.NONE).identical(res1_eq.to_dataset(bch.ReduceType.NONE)),
+            "created with identical settings so should be equal",
+        )
+
+        self.assertFalse(
+            res1.to_dataset(bch.ReduceType.NONE).identical(res2.to_dataset(bch.ReduceType.NONE)),
+            "different number of repeats so should not be equal",
+        )
+
+        res1_ds = res1.to_dataset(bch.ReduceType.SQUEEZE)
+        res2_ds = res2.to_dataset(bch.ReduceType.REDUCE)
+
+        self.assertFalse(
+            res1_ds.identical(res2_ds), "should not be equal because of std_dev column"
+        )
+
+        res2_ds = res2_ds.drop_vars("distance_std")
+        res2_ds = res2_ds.drop_vars("sample_noise_std")
+
+        print(res1_ds)
+        print(res2_ds)
+        print(res1_ds["distance"].attrs)
+        print(res2_ds["distance"].attrs)
+
+        self.assertTrue(
+            res1_ds.equals(res2_ds), "should be equal because of removed std_dev column"
+        )
+
+        self.assertTrue(
+            res1_ds.identical(res2_ds), "should be equal because of removed std_dev column"
+        )