Add confidence interval and standard error of the score

Closes #84
kiudee · Aug 24, 2020 · 4215024 · 4215024 · Claes1981 · Aug 25, 2020
1 parent 5d156ad
commit 4215024
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -6,6 +6,8 @@ History
 -------------------------
 * Improve default parameters to be slightly more robust for most use cases and
   be more in line with what a user might expect.
+* Add confidence interval and standard error of the score of the estimated
+  global optimum to the logging output
 * Fix debug output being spammed by other libraries.
 
 0.6.0-beta.0 (2020-08-24)

diff --git a/tune/cli.py b/tune/cli.py
@@ -11,6 +11,7 @@
 import numpy as np
 from atomicwrites import AtomicWriter
 from bask.optimizer import Optimizer
+from scipy.special import erfinv
 from skopt.utils import create_result
 
 from tune.db_workers import TuningClient, TuningServer
@@ -353,9 +354,21 @@ def local(  # noqa: C901
             try:
                 best_point, best_value = expected_ucb(result_object, alpha=0.0)
                 best_point_dict = dict(zip(param_ranges.keys(), best_point))
+                _, best_std = opt.gp.predict(
+                    opt.space.transform([best_point]), return_std=True
+                )
                 root_logger.info(f"Current optimum:\n{best_point_dict}")
-                root_logger.info(f"Estimated value: {best_value}")
+                root_logger.info(
+                    f"Estimated value: {np.around(best_value, 4)} +- "
+                    f"{np.around(best_std, 4).item()}"
+                )
                 confidence_val = settings.get("confidence", confidence)
+                confidence_mult = erfinv(confidence_val) * np.sqrt(2)
+                root_logger.info(
+                    f"{confidence_val * 100}% confidence interval of the value: "
+                    f"({np.around(best_value - confidence_mult * best_std, 4).item()}, "
+                    f"{np.around(best_value + confidence_mult * best_std, 4).item()})"
+                )
                 confidence_out = confidence_intervals(
                     optimizer=opt,
                     param_names=list(param_ranges.keys()),
@@ -364,7 +377,8 @@ def local(  # noqa: C901
                     multimodal=False,
                 )
                 root_logger.info(
-                    f"{confidence_val*100}% confidence intervals:\n{confidence_out}"
+                    f"{confidence_val * 100}% confidence intervals of the parameters:"
+                    f"\n{confidence_out}"
                 )
             except ValueError:
                 root_logger.info(