Merge pull request #193 from kiudee/91_pass_points

Allow the user to pass points manually

docs/faq.myst

@@ -105,6 +105,31 @@ It is possible, but it will bias the estimated Elo values to slightly more
 extreme ones. This could lead the model to temporarily over-/underevalute
 certain regions until enough new data points were collected.
 
+### How can I pass my own points to the tuner to evaluate?
+```{note}
+:class: margin
+The parameter values have to be within the bounds specified in the config file.
+```
+Starting with 0.9.2, you can pass your own points to the tuner to evaluate.
+To do this, you need to create a .csv file using the ``--evaluate-points``
+option (``-p`` for short).
+Each row in the file should contain the parameters
+in the same order as specified in the config file of the tune.
+It is also possible to add an additional integer column to the file, which
+will indicate the number of rounds to run each point for.
+
+Here is an example of how a .csv file for a three parameter tune could look
+like:
+
+    0.3,125,0.0,25
+    0.4,1000,0.5,50
+    0.5,1000,0.0,50
+
+The first point would be evaluated using 25 rounds (50 games), while the
+second and third point would be evaluated using 50 rounds (100 games).
+
+
+
 ## Problems while tuning
 
 ### The computational overhead of the tuner has become too high? What can I do?

docs/parameters.myst

@@ -219,6 +219,14 @@ fitting process:
   - The number of random points to consider as possible next point. Less points
     reduce the computation time per iteration, but reduce the coverage of the
     space. [default: 500]
+* -
+  - `-p --evaluate-points CSVFILE`
+  - Evaluate the given points first, before continuing with the points selected
+    by the tuner. The points are given as the rows of a CSV file with the
+    following format:
+    `x1,x2,...,xn[,samplesize]`. The first row should *not* be a header.
+    The last column is optional and contains the number of rounds to run each
+    point for. If not given, the default is whatever `"rounds"` is set to..
 ```
 
 ## Match-related parameters

tune/cli.py

@@ -18,6 +18,7 @@
     initialize_data,
     initialize_optimizer,
     is_debug_log,
+    load_points_to_evaluate,
     parse_experiment_result,
     plot_results,
     print_results,
@@ -257,6 +258,16 @@ def run_server(verbose, logfile, command, experiment_file, dbconfig):
     "the coverage of the space.",
     show_default=True,
 )
+@click.option(
+    "-p",
+    "--evaluate-points",
+    default=None,
+    type=click.Path(exists=False),
+    help="Path to a .csv file (without header row) with points to evaluate. An optional"
+    " last column can be used to request a different number of rounds for each "
+    "point.",
+    show_default=False,
+)
 @click.option(
     "--plot-every",
     default=1,
@@ -321,6 +332,7 @@ def local(  # noqa: C901
     acq_function_samples=1,
     confidence=0.9,
     data_path=None,
+    evaluate_points=None,
     gp_burnin=5,
     gp_samples=300,
     gp_initial_burnin=100,
@@ -404,6 +416,12 @@ def local(  # noqa: C901
         gp_initial_samples=settings.get("gp_initial_samples", gp_initial_samples),
         gp_priors=gp_priors,
     )
+    extra_points = load_points_to_evaluate(
+        space=opt.space, csv_file=evaluate_points, rounds=settings.get("rounds", 10),
+    )
+    root_logger.debug(
+        f"Loaded {len(extra_points)} extra points to evaluate: {extra_points}"
+    )
 
     # Main optimization loop:
     while True:
@@ -439,8 +457,22 @@ def local(  # noqa: C901
                     current_iteration=iteration,
                 )
 
-        # Ask optimizer for next point:
-        point = opt.ask()
+        used_extra_point = False
+        # If there are extra points to evaluate, evaluate them first in FIFO order:
+        if len(extra_points) > 0:
+            point, n_rounds = extra_points.pop(0)
+            # Log that we are evaluating the extra point:
+            root_logger.info(
+                f"Evaluating extra point {dict(zip(param_ranges.keys(), point))} for "
+                f"{n_rounds} rounds."
+            )
+            used_extra_point = True
+        else:
+            # Ask optimizer for next point:
+            point = opt.ask()
+            n_rounds = settings.get("rounds", 10)
+        match_settings = settings.copy()
+        match_settings["rounds"] = n_rounds
         point_dict = dict(zip(param_ranges.keys(), point))
         root_logger.info("Testing {}".format(point_dict))
 
@@ -454,7 +486,7 @@ def local(  # noqa: C901
         now = datetime.now()
         out_exp = []
         out_all = []
-        for output_line in run_match(**settings):
+        for output_line in run_match(**match_settings):
             line = output_line.rstrip()
             is_debug = is_debug_log(line)
             if is_debug and verbose > 2:
@@ -489,6 +521,9 @@ def local(  # noqa: C901
             gp_initial_burnin=settings.get("gp_initial_burnin", gp_initial_burnin),
             gp_initial_samples=settings.get("gp_initial_samples", gp_initial_samples),
         )
+        # If we used an extra point, we need to reset n_initial_points of the optimizer:
+        if used_extra_point:
+            opt._n_initial_points += 1
 
         # Update data structures and persist to disk:
         X.append(point)

tune/local.py

@@ -6,11 +6,12 @@
 import time
 from datetime import datetime
 from logging import Logger
-from typing import Callable, List, Optional, Sequence, Tuple, Union
+from typing import Callable, List, Optional, Sequence, TextIO, Tuple, Union
 
 import dill
 import matplotlib.pyplot as plt
 import numpy as np
+import pandas as pd
 from bask import Optimizer
 from matplotlib.transforms import Bbox
 from numpy.random import RandomState
@@ -173,6 +174,69 @@ def setup_logger(verbose: int = 0, logfile: str = "log.txt") -> Logger:
     return logger
 
 
+def load_points_to_evaluate(
+    space: Space, csv_file: Optional[TextIO] = None, rounds: int = 10
+) -> List[Tuple[List, int]]:
+    """Load extra points to evaluate from a csv file.
+
+    Parameters
+    ----------
+    space : Space
+        Optimization space containing the parameters to tune.
+    csv_file : TextIO, optional (default=None)
+        Comma-separated text file containing the points to evaluate. If an extra column
+        is present, it will be used as the number of rounds to play.
+    rounds : int, optional (default=10)
+        Number of rounds to play for each point. Will be used if no extra column is
+        present.
+
+    Returns
+    -------
+    List[Tuple[List, int]]
+        List of points to evaluate and number of rounds to play each point.
+
+    Raises
+    ------
+    ValueError
+        If the values in the csv file are out of the optimization bounds or if the
+        number of columns is not equal to the number of parameters (+1 in the case the
+        sample size is provided).
+    """
+
+    if csv_file is None:
+        # No file given, do not load any points:
+        return []
+
+    # Open csv file using pandas:
+    df = pd.read_csv(csv_file, header=None)
+
+    # First check if number of columns is correct:
+    n_dim = len(space.dimensions)
+    if len(df.columns) not in (n_dim, n_dim + 1):
+        raise ValueError(
+            f"Number of columns in csv file ({len(df.columns)}) does not match number"
+            f" of dimensions ({n_dim})."
+        )
+
+    # Check if the given points are within the lower and upper bounds of the space:
+    for i, dim in enumerate(space.dimensions):
+        if not (df.iloc[:, i].between(dim.low, dim.high).all()):
+            raise ValueError(
+                "Some points in the csv file are outside of the specified bounds."
+                " Please check the csv file and your configuration."
+            )
+
+    # If there is an extra column, extract it as the number of rounds for each point:
+    if len(df.columns) == n_dim + 1:
+        rounds_column = df.iloc[:, n_dim].values
+        df = df.iloc[:, :n_dim]
+    else:
+        rounds_column = np.full(len(df), rounds)
+
+    # All points are within the bounds, add them to the list of points to evaluate:
+    return [(x, r) for x, r in zip(df.values.tolist(), rounds_column)]
+
+
 def reduce_ranges(
     X: Sequence[list], y: Sequence[float], noise: Sequence[float], space: Space
 ) -> Tuple[bool, List[list], List[float], List[float]]: