Skip to content

Commit

Permalink
Merge pull request #86 from rodrigo-arenas/0.8.X
Browse files Browse the repository at this point in the history 
Graceful training stopping
  • Loading branch information
@rodrigo-arenas
rodrigo-arenas committed Dec 20, 2021
2 parents 1d67398 + 4adadcc commit fa1b928
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 129 deletions.
16 changes: 10 additions & 6 deletions docs/release_notes.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,14 @@ Features:

* Classes :class:`~sklearn_genetic.GASearchCV` and :class:`~sklearn_genetic.GAFeatureSelectionCV`
now support multi-metric evaluation the same way scikit-learn does,
you will see this reflected on the `logbook` and `cv_results` objects, where now you get results for each metric.
you will see this reflected on the `logbook` and `cv_results_` objects, where now you get results for each metric.
As in scikit-learn, if multi-metric is used, the `refit` parameter must be a str specifying the metric to evaluate the cv-scores.
See more in the :class:`~sklearn_genetic.GASearchCV` and :class:`~sklearn_genetic.GAFeatureSelectionCV` API documentation.

^^^^^
Docs:
^^^^^

* A new notebook called Iris_multimetric was added to showcase the new multi-metric capabilities.
* Training gracefully stops if interrupted by some of these exceptions:
``KeyboardInterrupt``, ``SystemExit``, ``StopIteration``.
When one of these exceptions is raised, the model finishes the current generation and saves the current
best model. It only works if at least one generation has been completed.

^^^^^^^^^^^^
API Changes:
Expand All @@ -39,6 +38,11 @@ API Changes:

- mutation_probability from 0.1 to 0.2

^^^^^
Docs:
^^^^^

* A new notebook called Iris_multimetric was added to showcase the new multi-metric capabilities.

What's new in 0.7.0
-------------------
Expand Down
266 changes: 145 additions & 121 deletions sklearn_genetic/algorithms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def eaSimple(
Number of generations used.
"""

stored_exception = None
callbacks_start_args = {
"callbacks": callbacks,
"record": None,
Expand Down Expand Up @@ -125,50 +125,57 @@ def eaSimple(

# Begin the generational process
for gen in range(1, ngen + 1):
# Select the next generation individuals
offspring = toolbox.select(population, len(population) - hof_size)

# Vary the pool of individuals
offspring = varAnd(offspring, toolbox, cxpb, mutpb)

# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit

if estimator.elitism:
offspring.extend(halloffame.items)

# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)

# Replace the current population by the offspring
population[:] = offspring

# Append the current generation statistics to the logbook
record = stats.compile(population) if stats else {}
if isinstance(record["fitness"], np.ndarray):
record = {key: value[0] for key, value in record.items()}

logbook.record(gen=gen, nevals=len(invalid_ind), **record)

if verbose:
print(logbook.stream)

callbacks_step_args = {
"callbacks": callbacks,
"record": record,
"logbook": logbook,
"estimator": estimator,
"method": "on_step",
}

# Check if any of the callbacks conditions are True to stop the iteration
if eval_callbacks(**callbacks_step_args):
print("INFO: Stopping the algorithm")
break
try:
# Select the next generation individuals
offspring = toolbox.select(population, len(population) - hof_size)

# Vary the pool of individuals
offspring = varAnd(offspring, toolbox, cxpb, mutpb)

# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit

if estimator.elitism:
offspring.extend(halloffame.items)

# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)

# Replace the current population by the offspring
population[:] = offspring

# Append the current generation statistics to the logbook
record = stats.compile(population) if stats else {}
if isinstance(record["fitness"], np.ndarray):
record = {key: value[0] for key, value in record.items()}

logbook.record(gen=gen, nevals=len(invalid_ind), **record)

if verbose:
print(logbook.stream)

callbacks_step_args = {
"callbacks": callbacks,
"record": record,
"logbook": logbook,
"estimator": estimator,
"method": "on_step",
}

# Check if any of the callbacks conditions are True to stop the iteration
if eval_callbacks(**callbacks_step_args) or stored_exception:
if stored_exception:
print(
f"{stored_exception}\nsklearn-genetic-opt closed prematurely. Will use the current best model."
)
print("INFO: Stopping the algorithm")
break
except (KeyboardInterrupt, SystemExit, StopIteration) as e:
stored_exception = e

n_gen = gen + 1

Expand Down Expand Up @@ -253,7 +260,7 @@ def eaMuPlusLambda(
Number of generations used.
"""

stored_exception = None
callbacks_start_args = {
"callbacks": callbacks,
"record": None,
Expand Down Expand Up @@ -310,43 +317,51 @@ def eaMuPlusLambda(

# Begin the generational process
for gen in range(1, ngen + 1):
# Vary the population
offspring = varOr(population, toolbox, lambda_, cxpb, mutpb)

# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit

# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)

# Select the next generation population
population[:] = toolbox.select(population + offspring, mu)

# Update the statistics with the new population
record = stats.compile(population) if stats is not None else {}
if isinstance(record["fitness"], np.ndarray):
record = {key: value[0] for key, value in record.items()}

logbook.record(gen=gen, nevals=len(invalid_ind), **record)

if verbose:
print(logbook.stream)

callbacks_step_args = {
"callbacks": callbacks,
"record": record,
"logbook": logbook,
"estimator": estimator,
"method": "on_step",
}

if eval_callbacks(**callbacks_step_args):
print("INFO: Stopping the algorithm")
break
try:
# Vary the population
offspring = varOr(population, toolbox, lambda_, cxpb, mutpb)

# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit

# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)

# Select the next generation population
population[:] = toolbox.select(population + offspring, mu)

# Update the statistics with the new population
record = stats.compile(population) if stats is not None else {}
if isinstance(record["fitness"], np.ndarray):
record = {key: value[0] for key, value in record.items()}

logbook.record(gen=gen, nevals=len(invalid_ind), **record)

if verbose:
print(logbook.stream)

callbacks_step_args = {
"callbacks": callbacks,
"record": record,
"logbook": logbook,
"estimator": estimator,
"method": "on_step",
}

if eval_callbacks(**callbacks_step_args) or stored_exception:
if stored_exception:
print(
f"{stored_exception}\nsklearn-genetic-opt closed prematurely. Will use the current best model."
)
print("INFO: Stopping the algorithm")
break

except (KeyboardInterrupt, SystemExit, StopIteration) as e:
stored_exception = e

n_gen = gen + 1

Expand Down Expand Up @@ -433,6 +448,7 @@ def eaMuCommaLambda(
"""
assert lambda_ >= mu, "lambda must be greater or equal to mu."

stored_exception = None
callbacks_start_args = {
"callbacks": callbacks,
"record": None,
Expand Down Expand Up @@ -489,44 +505,52 @@ def eaMuCommaLambda(

# Begin the generational process
for gen in range(1, ngen + 1):
# Vary the population
offspring = varOr(population, toolbox, lambda_, cxpb, mutpb)

# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit

# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)

# Select the next generation population
population[:] = toolbox.select(offspring, mu)

# Update the statistics with the new population
record = stats.compile(population) if stats is not None else {}
if isinstance(record["fitness"], np.ndarray):
record = {key: value[0] for key, value in record.items()}

logbook.record(gen=gen, nevals=len(invalid_ind), **record)

if verbose:
print(logbook.stream)

callbacks_step_args = {
"callbacks": callbacks,
"record": record,
"logbook": logbook,
"estimator": estimator,
"method": "on_step",
}

# Check if any of the callbacks conditions are True to stop the iteration
if eval_callbacks(**callbacks_step_args):
print("INFO: Stopping the algorithm")
break
try:
# Vary the population
offspring = varOr(population, toolbox, lambda_, cxpb, mutpb)

# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit

# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)

# Select the next generation population
population[:] = toolbox.select(offspring, mu)

# Update the statistics with the new population
record = stats.compile(population) if stats is not None else {}
if isinstance(record["fitness"], np.ndarray):
record = {key: value[0] for key, value in record.items()}

logbook.record(gen=gen, nevals=len(invalid_ind), **record)

if verbose:
print(logbook.stream)

callbacks_step_args = {
"callbacks": callbacks,
"record": record,
"logbook": logbook,
"estimator": estimator,
"method": "on_step",
}

# Check if any of the callbacks conditions are True to stop the iteration
if eval_callbacks(**callbacks_step_args) or stored_exception:
if stored_exception:
print(
f"{stored_exception}\nsklearn-genetic-opt closed prematurely. Will use the current best model."
)
print("INFO: Stopping the algorithm")
break

except (KeyboardInterrupt, SystemExit, StopIteration) as e:
stored_exception = e

n_gen = gen + 1

Expand Down
4 changes: 2 additions & 2 deletions sklearn_genetic/tests/test_feature_selection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,13 +324,13 @@ def test_wrong_algorithm():

def test_expected_ga_max_features():
clf = SGDClassifier(loss="log", fit_intercept=True)
generations = 10
generations = 8
max_features = 6
evolved_estimator = GAFeatureSelectionCV(
clf,
cv=3,
scoring="accuracy",
population_size=6,
population_size=20,
generations=generations,
tournament_size=3,
elitism=False,
Expand Down

0 comments on commit fa1b928

Please sign in to comment.