Skip to content

Commit

Permalink
PLANNER-1312 Two tests take too long + new blueprint type
Browse files Browse the repository at this point in the history
  • Loading branch information
@ge0ffrey
ge0ffrey committed Nov 22, 2018
1 parent cc1751b commit e785325
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 91 deletions.
9 changes: 9 additions & 0 deletions .../main/java/org/optaplanner/benchmark/config/blueprint/SolverBenchmarkBluePrintConfig.java
View file
Expand Up @@ -51,4 +51,13 @@ protected void validate() {
}
}

// ************************************************************************
// With methods
// ************************************************************************

public SolverBenchmarkBluePrintConfig withSolverBenchmarkBluePrintType(SolverBenchmarkBluePrintType solverBenchmarkBluePrintType) {
this.solverBenchmarkBluePrintType = solverBenchmarkBluePrintType;
return this;
}

}
43 changes: 33 additions & 10 deletions ...rc/main/java/org/optaplanner/benchmark/config/blueprint/SolverBenchmarkBluePrintType.java
View file
Expand Up @@ -28,6 +28,10 @@
import org.optaplanner.core.config.solver.SolverConfig;

public enum SolverBenchmarkBluePrintType {
/*
* Run the default {@link ConstructionHeuristicType} with and without the default {@link LocalSearchType}.
*/
CONSTRUCTION_HEURISTIC_WITH_AND_WITHOUT_LOCAL_SEARCH,
/**
* Run every {@link ConstructionHeuristicType}.
*/
Expand All @@ -43,6 +47,8 @@ public enum SolverBenchmarkBluePrintType {

protected List<SolverBenchmarkConfig> buildSolverBenchmarkConfigList() {
switch (this) {
case CONSTRUCTION_HEURISTIC_WITH_AND_WITHOUT_LOCAL_SEARCH:
return buildConstructionHeuristicWithAndWithoutLocalSearch();
case EVERY_CONSTRUCTION_HEURISTIC_TYPE:
return buildEveryConstructionHeuristicType();
case EVERY_LOCAL_SEARCH_TYPE:
Expand All @@ -55,11 +61,18 @@ protected List<SolverBenchmarkConfig> buildSolverBenchmarkConfigList() {
}
}

private List<SolverBenchmarkConfig> buildConstructionHeuristicWithAndWithoutLocalSearch() {
List<SolverBenchmarkConfig> solverBenchmarkConfigList = new ArrayList<>(2);
solverBenchmarkConfigList.add(buildSolverBenchmarkConfig(null, false, null));
solverBenchmarkConfigList.add(buildSolverBenchmarkConfig(null, true, null));
return solverBenchmarkConfigList;
}

private List<SolverBenchmarkConfig> buildEveryConstructionHeuristicType() {
ConstructionHeuristicType[] chTypes = ConstructionHeuristicType.getBluePrintTypes();
List<SolverBenchmarkConfig> solverBenchmarkConfigList = new ArrayList<>(chTypes.length);
for (ConstructionHeuristicType chType : chTypes) {
solverBenchmarkConfigList.add(buildSolverBenchmarkConfig(chType, null));
solverBenchmarkConfigList.add(buildSolverBenchmarkConfig(chType, false, null));
}
return solverBenchmarkConfigList;
}
Expand All @@ -68,7 +81,7 @@ private List<SolverBenchmarkConfig> buildEveryLocalSearchType() {
LocalSearchType[] lsTypes = LocalSearchType.getBluePrintTypes();
List<SolverBenchmarkConfig> solverBenchmarkConfigList = new ArrayList<>(lsTypes.length);
for (LocalSearchType lsType : lsTypes) {
solverBenchmarkConfigList.add(buildSolverBenchmarkConfig(null, lsType));
solverBenchmarkConfigList.add(buildSolverBenchmarkConfig(null, true, lsType));
}
return solverBenchmarkConfigList;
}
Expand All @@ -80,18 +93,26 @@ private List<SolverBenchmarkConfig> buildEveryConstructionHeuristicTypeWithEvery
chTypes.length * lsTypes.length);
for (ConstructionHeuristicType chType : chTypes) {
for (LocalSearchType lsType : lsTypes) {
solverBenchmarkConfigList.add(buildSolverBenchmarkConfig(chType, lsType));
solverBenchmarkConfigList.add(buildSolverBenchmarkConfig(chType, true, lsType));
}
}
return solverBenchmarkConfigList;
}

protected SolverBenchmarkConfig buildSolverBenchmarkConfig(
ConstructionHeuristicType constructionHeuristicType, LocalSearchType localSearchType) {
protected SolverBenchmarkConfig buildSolverBenchmarkConfig(ConstructionHeuristicType constructionHeuristicType,
boolean localSearchEnabled, LocalSearchType localSearchType) {
SolverBenchmarkConfig solverBenchmarkConfig = new SolverBenchmarkConfig();
String name = localSearchType == null ? constructionHeuristicType.name() :
constructionHeuristicType == null ? localSearchType.name() :
constructionHeuristicType.name() + "-" + localSearchType.name();
String constructionHeuristicName = constructionHeuristicType == null
? "Construction Heuristic" : constructionHeuristicType.name();
String name;
if (!localSearchEnabled) {
name = constructionHeuristicName;
} else {
String localSearchName = localSearchType == null
? "Local Search" : localSearchType.name();
name = constructionHeuristicType == null ? localSearchName
: constructionHeuristicName + " - " + localSearchName;
}
solverBenchmarkConfig.setName(name);
SolverConfig solverConfig = new SolverConfig();
List<PhaseConfig> phaseConfigList = new ArrayList<>(2);
Expand All @@ -100,9 +121,11 @@ protected SolverBenchmarkConfig buildSolverBenchmarkConfig(
constructionHeuristicPhaseConfig.setConstructionHeuristicType(constructionHeuristicType);
}
phaseConfigList.add(constructionHeuristicPhaseConfig);
if (localSearchType != null) {
if (localSearchEnabled) {
LocalSearchPhaseConfig localSearchPhaseConfig = new LocalSearchPhaseConfig();
localSearchPhaseConfig.setLocalSearchType(localSearchType);
if (localSearchType != null) {
localSearchPhaseConfig.setLocalSearchType(localSearchType);
}
phaseConfigList.add(localSearchPhaseConfig);
}
solverConfig.setPhaseConfigList(phaseConfigList);
Expand Down
2 changes: 2 additions & 0 deletions .../src/main/asciidoc/BenchmarkingAndTweaking/BenchmarkingAndTweaking-chapter.adoc
View file
Expand Up @@ -339,6 +339,8 @@ To quickly configure and run a benchmark for typical solver configs, use a `solv

The following ``SolverBenchmarkBluePrintType``s are supported:

* ``CONSTRUCTION_HEURISTIC_WITH_AND_WITHOUT_LOCAL_SEARCH``: Run the default Construction Heuristic type with and without the default Local Search type.

* ``EVERY_CONSTRUCTION_HEURISTIC_TYPE``: Run every Construction Heuristic type (First Fit, First Fit Decreasing, Cheapest Insertion, ...).

* ``EVERY_LOCAL_SEARCH_TYPE``: Run every Local Search type (Tabu Search, Late Acceptance, ...) with the default Construction Heuristic.
Expand Down
73 changes: 20 additions & 53 deletions ...nner-examples/src/test/java/org/optaplanner/examples/common/app/PlannerBenchmarkTest.java
View file
Expand Up @@ -17,14 +17,19 @@
package org.optaplanner.examples.common.app;

import java.io.File;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collector;

import org.optaplanner.benchmark.api.PlannerBenchmark;
import org.optaplanner.benchmark.api.PlannerBenchmarkFactory;
import org.optaplanner.benchmark.config.PlannerBenchmarkConfig;
import org.optaplanner.benchmark.config.ProblemBenchmarksConfig;
import org.optaplanner.benchmark.config.SolverBenchmarkConfig;
import org.optaplanner.benchmark.config.blueprint.SolverBenchmarkBluePrintConfig;
import org.optaplanner.benchmark.config.blueprint.SolverBenchmarkBluePrintType;
import org.optaplanner.core.api.solver.SolverFactory;
import org.optaplanner.core.config.solver.termination.TerminationConfig;
import org.optaplanner.core.config.util.ConfigUtils;

Expand All @@ -41,64 +46,26 @@
*/
public abstract class PlannerBenchmarkTest extends LoggingTest {

private static final int MAXIMUM_SOLVER_BENCHMARK_SIZE = 6;
private static final long WARM_UP_SECONDS_SPENT = 5L;
private static final long MAXIMUM_SECONDS_SPENT = 30L;
private static final long WARM_UP_SECONDS_SPENT = 2L;
private static final long MAXIMUM_SECONDS_SPENT = 8L;

protected abstract String createBenchmarkConfigResource();
protected abstract String createSolverConfigResource();

protected void runBenchmarkTest(File unsolvedDataFile) {
PlannerBenchmarkFactory plannerBenchmarkFactory = buildPlannerBenchmarkFactory(unsolvedDataFile);
PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark();
plannerBenchmark.benchmark();
}

protected PlannerBenchmarkFactory buildPlannerBenchmarkFactory(File unsolvedDataFile) {
String benchmarkConfigResource = createBenchmarkConfigResource();
PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource(benchmarkConfigResource);
protected PlannerBenchmarkFactory buildPlannerBenchmarkFactory() {
SolverFactory<Object> solverFactory = SolverFactory.createFromXmlResource(createSolverConfigResource());
File benchmarkDirectory = new File("target/test/data");
PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverFactory(solverFactory, benchmarkDirectory);
PlannerBenchmarkConfig plannerBenchmarkConfig = benchmarkFactory.getPlannerBenchmarkConfig();
String benchmarkDirectoryPath = plannerBenchmarkConfig.getBenchmarkDirectory().getPath();
// On Windows, getPath() contains backslashes instead of normal slashes
String prefix = "local" + File.separator + "data" + File.separator;
if (!benchmarkDirectoryPath.startsWith(prefix)) {
throw new IllegalStateException("The benchmarkDirectoryPath (" + benchmarkDirectoryPath
+ ") should start with prefix (" + prefix + ")");
}
plannerBenchmarkConfig.setBenchmarkDirectory(new File(benchmarkDirectoryPath.replace(prefix,
"target" + File.separator + "test" + File.separator + "data" + File.separator)));
plannerBenchmarkConfig.setWarmUpHoursSpentLimit(0L);
plannerBenchmarkConfig.setWarmUpMinutesSpentLimit(0L);
plannerBenchmarkConfig.setWarmUpSecondsSpentLimit(WARM_UP_SECONDS_SPENT);
plannerBenchmarkConfig.setWarmUpMillisecondsSpentLimit(0L);
List<SolverBenchmarkConfig> solverBenchmarkConfigList = plannerBenchmarkConfig.getSolverBenchmarkConfigList();
if (ConfigUtils.isEmptyCollection(solverBenchmarkConfigList)) {
throw new IllegalStateException("The benchmarkConfigResource (" + benchmarkConfigResource
+ ") should have at least 1 solverBenchmarkConfig.");
}
if (solverBenchmarkConfigList.size() > MAXIMUM_SOLVER_BENCHMARK_SIZE) {
solverBenchmarkConfigList = solverBenchmarkConfigList.subList(0, MAXIMUM_SOLVER_BENCHMARK_SIZE);
plannerBenchmarkConfig.setSolverBenchmarkConfigList(solverBenchmarkConfigList);
}
long maximumSecondsSpentPerSolverBenchmark = MAXIMUM_SECONDS_SPENT / solverBenchmarkConfigList.size();
plannerBenchmarkConfig.setSolverBenchmarkConfigList(Collections.emptyList());
plannerBenchmarkConfig.setSolverBenchmarkBluePrintConfigList(Collections.singletonList(
new SolverBenchmarkBluePrintConfig().withSolverBenchmarkBluePrintType(
SolverBenchmarkBluePrintType.CONSTRUCTION_HEURISTIC_WITH_AND_WITHOUT_LOCAL_SEARCH)));

long maximumSecondsSpentPerSolverBenchmark = MAXIMUM_SECONDS_SPENT / 2;
SolverBenchmarkConfig inheritedSolverBenchmarkConfig = plannerBenchmarkConfig.getInheritedSolverBenchmarkConfig();
if (inheritedSolverBenchmarkConfig != null) {
ProblemBenchmarksConfig problemBenchmarksConfig = inheritedSolverBenchmarkConfig.getProblemBenchmarksConfig();
if (problemBenchmarksConfig == null) {
problemBenchmarksConfig = new ProblemBenchmarksConfig();
inheritedSolverBenchmarkConfig.setProblemBenchmarksConfig(problemBenchmarksConfig);
}
problemBenchmarksConfig.setInputSolutionFileList(
Collections.singletonList(unsolvedDataFile));
inheritedSolverBenchmarkConfig.getSolverConfig().setTerminationConfig(
new TerminationConfig().withSecondsSpentLimit(maximumSecondsSpentPerSolverBenchmark));
}
for (SolverBenchmarkConfig solverBenchmarkConfig : solverBenchmarkConfigList) {
ProblemBenchmarksConfig problemBenchmarksConfig = solverBenchmarkConfig.getProblemBenchmarksConfig();
if (problemBenchmarksConfig != null) {
problemBenchmarksConfig.setInputSolutionFileList(null);
}
solverBenchmarkConfig.getSolverConfig().setTerminationConfig(new TerminationConfig());
}
inheritedSolverBenchmarkConfig.getSolverConfig().setTerminationConfig(
new TerminationConfig().withSecondsSpentLimit(maximumSecondsSpentPerSolverBenchmark));
return benchmarkFactory;
}

Expand Down
4 changes: 2 additions & 2 deletions ...java/org/optaplanner/examples/meetingscheduling/app/MeetingSchedulingPerformanceTest.java
View file
Expand Up @@ -41,13 +41,13 @@ protected MeetingSchedulingApp createCommonApp() {
@Test(timeout = 600000)
public void solveModel() {
File unsolvedDataFile = new File("data/meetingscheduling/unsolved/50meetings-160timegrains-5rooms.xlsx");
runSpeedTest(unsolvedDataFile, "-19hard/-115medium/-4046soft");
runSpeedTest(unsolvedDataFile, "-35hard/-86medium/-6090soft");
}

@Test(timeout = 600000)
public void solveModelFastAssert() {
File unsolvedDataFile = new File("data/meetingscheduling/unsolved/50meetings-160timegrains-5rooms.xlsx");
runSpeedTest(unsolvedDataFile, "-29hard/-70medium/-3399soft", EnvironmentMode.FAST_ASSERT);
runSpeedTest(unsolvedDataFile, "-36hard/-64medium/-5921soft", EnvironmentMode.FAST_ASSERT);
}

}
17 changes: 12 additions & 5 deletions ...amples/src/test/java/org/optaplanner/examples/nqueens/app/BrokenNQueensBenchmarkTest.java
View file
Expand Up @@ -19,21 +19,24 @@
import java.io.File;

import org.junit.Test;
import org.optaplanner.benchmark.api.PlannerBenchmark;
import org.optaplanner.benchmark.api.PlannerBenchmarkException;
import org.optaplanner.benchmark.api.PlannerBenchmarkFactory;
import org.optaplanner.benchmark.config.PlannerBenchmarkConfig;
import org.optaplanner.examples.common.app.PlannerBenchmarkTest;
import org.optaplanner.examples.nqueens.domain.NQueens;
import org.optaplanner.persistence.xstream.impl.domain.solution.XStreamSolutionFileIO;

public class BrokenNQueensBenchmarkTest extends PlannerBenchmarkTest {

@Override
protected String createBenchmarkConfigResource() {
return "org/optaplanner/examples/nqueens/benchmark/nqueensBenchmarkConfig.xml";
protected String createSolverConfigResource() {
return NQueensApp.SOLVER_CONFIG;
}

@Override
protected PlannerBenchmarkFactory buildPlannerBenchmarkFactory(File unsolvedDataFile) {
PlannerBenchmarkFactory benchmarkFactory = super.buildPlannerBenchmarkFactory(unsolvedDataFile);
protected PlannerBenchmarkFactory buildPlannerBenchmarkFactory() {
PlannerBenchmarkFactory benchmarkFactory = super.buildPlannerBenchmarkFactory();
PlannerBenchmarkConfig benchmarkConfig = benchmarkFactory.getPlannerBenchmarkConfig();
benchmarkConfig.setWarmUpSecondsSpentLimit(0L);
benchmarkConfig.getInheritedSolverBenchmarkConfig().getSolverConfig().getTerminationConfig()
Expand All @@ -47,7 +50,11 @@ protected PlannerBenchmarkFactory buildPlannerBenchmarkFactory(File unsolvedData

@Test(timeout = 100000, expected = PlannerBenchmarkException.class)
public void benchmarkBroken8queens() {
runBenchmarkTest(new File("data/nqueens/unsolved/8queens.xml"));
NQueens problem = new XStreamSolutionFileIO<NQueens>(NQueens.class)
.read(new File("data/nqueens/unsolved/8queens.xml"));
PlannerBenchmarkFactory plannerBenchmarkFactory = buildPlannerBenchmarkFactory();
PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(problem);
plannerBenchmark.benchmark();
}

}

0 comments on commit e785325

Please sign in to comment.