Merge pull request #75 from alan-turing-institute/selection-heuristic

Allow user to specify a selection heuristic

src/MLJTuning.jl

@@ -9,6 +9,9 @@ export TunedModel
 # defined in strategies/:
 export Explicit, Grid, RandomSearch
 
+# defined in selection_heuristics/:
+export NaiveSelection
+
 # defined in learning_curves.jl:
 export learning_curve!, learning_curve
 
@@ -37,6 +40,7 @@ const DEFAULT_N = 10 # for when `default_n` is not implemented
 
 include("utilities.jl")
 include("tuning_strategy_interface.jl")
+include("selection_heuristics.jl")
 include("tuned_models.jl")
 include("range_methods.jl")
 include("strategies/explicit.jl")

src/selection_heuristics.jl

@@ -0,0 +1,61 @@
+abstract type SelectionHeuristic end
+
+## HELPERS
+
+measure_adjusted_weights(weights, measures) =
+    if weights isa Nothing
+        vcat([signature(measures[1]), ], zeros(length(measures) - 1))
+    else
+        length(weights) == length(measures) ||
+            throw(DimensionMismatch(
+                "`OptimizeAggregatedMeasurement` heuristic "*
+                "is being applied to a list of measures whose length "*
+                "differs from that of the specified `weights`. "))
+        signature.(measures) .* weights
+    end
+
+
+## OPTIMIZE AGGREGATED MEASURE
+
+"""
+    NaiveSelection(; weights=nothing)
+
+Construct a common selection heuristic for use with `TunedModel` instances
+which only considers measurements aggregated over all samples (folds)
+in resampling.
+
+For each entry in the tuning history, one defines a penalty equal to
+the evaluations of the `measure` specified in the `TunedModel`
+instance, aggregated over all samples, and multiplied by `-1` if `measure`
+is a `:score`, and `+`` if it is a loss. The heuristic declares as
+"best" (optimal) the model whose corresponding entry has the lowest
+penalty.
+
+If `measure` is a vector, then the first element is used, unless
+per-measure `weights` are explicitly specified. Weights associated
+with measures that are neither `:loss` nor `:score` are reset to zero.
+
+"""
+struct NaiveSelection <: SelectionHeuristic
+    weights::Union{Nothing, Vector{Real}}
+end
+NaiveSelection(; weights=nothing) =
+    NaiveSelection(weights)
+
+
+function best(heuristic::NaiveSelection, history)
+    first_entry = history[1]
+    measures = first_entry.measure
+    weights = measure_adjusted_weights(heuristic.weights, measures)
+    measurements = [weights'*(h.measurement) for h in history]
+    measure = first(history).measure[1]
+    if orientation(measure) == :score
+                               measurements = -measurements
+
+    end
+    best_index = argmin(measurements)
+    return history[best_index]
+end
+
+MLJTuning.supports_heuristic(::Any, ::NaiveSelection) =
+    true

src/strategies/explicit.jl

@@ -1,6 +1,6 @@
 mutable struct Explicit <: TuningStrategy end
 
-mutable struct ExplicitState{R,N}
+struct ExplicitState{R,N}
     range::R               # a model-generating iterator
     next::Union{Nothing,N} # to hold output of `iterate(range)`
 end
@@ -14,7 +14,7 @@ function MLJTuning.setup(tuning::Explicit, model, range, verbosity)
 end
 
 # models! returns all available models in the range at once:
-function MLJTuning.models!(tuning::Explicit,
+function MLJTuning.models(tuning::Explicit,
                            model,
                            history,
                            state,
@@ -23,7 +23,7 @@ function MLJTuning.models!(tuning::Explicit,
 
     range, next  = state.range, state.next
 
-    next === nothing && return nothing
+    next === nothing && return nothing, state
 
     m, s = next
     models = [m, ]
@@ -39,9 +39,9 @@ function MLJTuning.models!(tuning::Explicit,
         next = iterate(range, s)
     end
 
-    state.next = next
+    new_state = ExplicitState(range, next)
 
-    return models
+    return models, new_state
 
 end
 

src/strategies/grid.jl

@@ -152,22 +152,16 @@ function setup(tuning::Grid, model, user_range, verbosity)
 
 end
 
-MLJTuning.models!(tuning::Grid,
-                  model,
-                  history,
-                  state,
-                  n_remaining,
-                  verbosity) =
-                      state.models[_length(history) + 1:end]
-
-function tuning_report(tuning::Grid, history, state)
-
-    plotting = plotting_report(state.fields, state.parameter_scales, history)
-
-    # todo: remove collects?
-    return (history=history, plotting=plotting)
-
-end
+MLJTuning.models(tuning::Grid,
+                 model,
+                 history,
+                 state,
+                 n_remaining,
+                 verbosity) =
+                     state.models[_length(history) + 1:end], state
+
+tuning_report(tuning::Grid, history, state) =
+    (plotting = plotting_report(state.fields, state.parameter_scales, history),)
 
 function default_n(tuning::Grid, user_range)
 

src/strategies/random_search.jl

@@ -116,20 +116,21 @@ setup(tuning::RandomSearch, model, user_range, verbosity) =
                               tuning.positive_unbounded,
                               tuning.other) |> collect
 
-function MLJTuning.models!(tuning::RandomSearch,
-                           model,
-                           history,
-                           state, # tuple of (field, sampler) pairs
-                           n_remaining,
-                           verbosity)
-    return map(1:n_remaining) do _
+function MLJTuning.models(tuning::RandomSearch,
+                          model,
+                          history,
+                          state, # tuple of (field, sampler) pairs
+                          n_remaining,
+                          verbosity)
+    new_models = map(1:n_remaining) do _
         clone = deepcopy(model)
         Random.shuffle!(tuning.rng, state)
         for (fld, s) in state
             recursive_setproperty!(clone, fld, rand(tuning.rng, s))
         end
         clone
     end
+    return new_models, state
 end
 
 function tuning_report(tuning::RandomSearch, history, field_sampler_pairs)
@@ -141,6 +142,6 @@ function tuning_report(tuning::RandomSearch, history, field_sampler_pairs)
 
     plotting = plotting_report(fields, parameter_scales, history)
 
-    return (history=history, plotting=plotting)
+    return (plotting=plotting,)
 
 end