wrong configuration breaking warmup should not break entire benchmark…

… + move warmUp method to Runner
kiegroup · May 21, 2015 · c5aa3b6 · c5aa3b6
1 parent 35a90d5
commit c5aa3b6
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 28 deletions.
diff --git a/...lanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/PlannerBenchmarkRunner.java b/...lanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/PlannerBenchmarkRunner.java
@@ -35,6 +35,8 @@
 import org.optaplanner.benchmark.impl.result.ProblemBenchmarkResult;
 import org.optaplanner.benchmark.impl.result.SingleBenchmarkResult;
 import org.optaplanner.benchmark.impl.result.SolverBenchmarkResult;
+import org.optaplanner.core.api.solver.Solver;
+import org.optaplanner.core.config.solver.termination.TerminationConfig;
 import org.optaplanner.core.config.util.ConfigUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -131,14 +133,41 @@ private void warmUp() {
                     it = unifiedProblemBenchmarkResultList.iterator();
                 }
                 ProblemBenchmarkResult problemBenchmarkResult = it.next();
-                timeLeft = problemBenchmarkResult.warmUp(startingTimeMillis, plannerBenchmarkResult.getWarmUpTimeMillisSpentLimit(), timeLeft);
+                timeLeft = warmUp(problemBenchmarkResult, startingTimeMillis, plannerBenchmarkResult.getWarmUpTimeMillisSpentLimit(), timeLeft);
             }
             logger.info("================================================================================");
             logger.info("Warm up ended");
             logger.info("================================================================================");
         }
     }
 
+    protected long warmUp(ProblemBenchmarkResult problemBenchmarkResult, long startingTimeMillis, long warmUpTimeMillisSpentLimit, long timeLeft) {
+        for (SingleBenchmarkResult singleBenchmarkResult : problemBenchmarkResult.getSingleBenchmarkResultList()) {
+            SolverBenchmarkResult solverBenchmarkResult = singleBenchmarkResult.getSolverBenchmarkResult();
+            TerminationConfig originalTerminationConfig = solverBenchmarkResult.getSolverConfig().getTerminationConfig();
+            TerminationConfig tmpTerminationConfig = originalTerminationConfig == null
+                    ? new TerminationConfig() : originalTerminationConfig.clone();
+            tmpTerminationConfig.shortenTimeMillisSpentLimit(timeLeft);
+            solverBenchmarkResult.getSolverConfig().setTerminationConfig(tmpTerminationConfig);
+
+            try {
+                Solver solver = solverBenchmarkResult.getSolverConfig().buildSolver();
+                solver.solve(problemBenchmarkResult.readPlanningProblem());
+            } catch (RuntimeException e) {
+                logger.error("The warmUp of singleBenchmark (" + singleBenchmarkResult.getName() + ") failed.", e);
+                // TODO update firstFailureSingleBenchmarkRunner (when warmups happen in a Runner too in parallel)
+            }
+
+            solverBenchmarkResult.getSolverConfig().setTerminationConfig(originalTerminationConfig);
+            long timeSpent = System.currentTimeMillis() - startingTimeMillis;
+            timeLeft = warmUpTimeMillisSpentLimit - timeSpent;
+            if (timeLeft <= 0L) {
+                return timeLeft;
+            }
+        }
+        return timeLeft;
+    }
+
     protected void runSingleBenchmarks() {
         Map<SingleBenchmarkRunner, Future<SingleBenchmarkRunner>> futureMap
                 = new HashMap<SingleBenchmarkRunner, Future<SingleBenchmarkRunner>>();

diff --git a/...benchmark/src/main/java/org/optaplanner/benchmark/impl/result/ProblemBenchmarkResult.java b/...benchmark/src/main/java/org/optaplanner/benchmark/impl/result/ProblemBenchmarkResult.java
@@ -263,28 +263,6 @@ public void makeDirs(File benchmarkReportDirectory) {
         }
     }
 
-    public long warmUp(long startingTimeMillis, long warmUpTimeMillisSpentLimit, long timeLeft) {
-        for (SingleBenchmarkResult singleBenchmarkResult : singleBenchmarkResultList) {
-            SolverBenchmarkResult solverBenchmarkResult = singleBenchmarkResult.getSolverBenchmarkResult();
-            TerminationConfig originalTerminationConfig = solverBenchmarkResult.getSolverConfig().getTerminationConfig();
-            TerminationConfig tmpTerminationConfig = originalTerminationConfig == null
-                    ? new TerminationConfig() : originalTerminationConfig.clone();
-            tmpTerminationConfig.shortenTimeMillisSpentLimit(timeLeft);
-            solverBenchmarkResult.getSolverConfig().setTerminationConfig(tmpTerminationConfig);
-
-            Solver solver = solverBenchmarkResult.getSolverConfig().buildSolver();
-            solver.solve(readPlanningProblem());
-
-            solverBenchmarkResult.getSolverConfig().setTerminationConfig(originalTerminationConfig);
-            long timeSpent = System.currentTimeMillis() - startingTimeMillis;
-            timeLeft = warmUpTimeMillisSpentLimit - timeSpent;
-            if (timeLeft <= 0L) {
-                return timeLeft;
-            }
-        }
-        return timeLeft;
-    }
-
     public Solution readPlanningProblem() {
         return solutionFileIO.read(inputSolutionFile);
     }

diff --git a/...er-distribution/src/main/assembly/filtered-resources/UpgradeFromPreviousVersionRecipe.txt b/...er-distribution/src/main/assembly/filtered-resources/UpgradeFromPreviousVersionRecipe.txt
@@ -2430,3 +2430,14 @@ After in *.java:
             return processName;
         }
     }
+
+[MINOR] Default parameter tweaked: a empty <constructionHeuristic>'s default changed from FIRST_FIT to ALLOCATE_ENTITY_FROM_QUEUE.
+If no entity difficulty comparison and no planning value strength comparison is defined, the behavior is exactly the same.
+Otherwise, the behaviour is FIRST_FIT_DECREASING or WEAKEST_FIT(_DECREASING).
+
+[MINOR] The solverBenchmarkBluePrintType ALL_CONSTRUCTION_HEURISTIC_TYPES has been renamed to EVERY_CONSTRUCTION_HEURISTIC_TYPE.
+The old name ALL_CONSTRUCTION_HEURISTIC_TYPES is deprecated and will be removed in 7.0.
+Before in *BenchmarkConfig.xml:
+    <solverBenchmarkBluePrintType>ALL_CONSTRUCTION_HEURISTIC_TYPES</solverBenchmarkBluePrintType>
+After in *BenchmarkConfig.xml:
+    <solverBenchmarkBluePrintType>EVERY_CONSTRUCTION_HEURISTIC_TYPE</solverBenchmarkBluePrintType>
diff --git a/...ain/docbook/en-US/Chapter-Benchmarking_and_tweaking/Chapter-Benchmarking_and_tweaking.xml b/...ain/docbook/en-US/Chapter-Benchmarking_and_tweaking/Chapter-Benchmarking_and_tweaking.xml
@@ -275,22 +275,39 @@
         &lt;scoreDrl&gt;org/optaplanner/examples/nqueens/solver/nQueensScoreRules.drl&lt;/scoreDrl&gt;
         &lt;initializingScoreTrend&gt;ONLY_DOWN&lt;/initializingScoreTrend&gt;
       &lt;/scoreDirectorFactory&gt;
+      &lt;termination&gt;
+        &lt;minutesSpentLimit&gt;1&lt;/minutesSpentLimit&gt;
+      &lt;/termination&gt;
     &lt;/solver&gt;
   &lt;/inheritedSolverBenchmark&gt;
 
   &lt;solverBenchmarkBluePrint&gt;
-    &lt;solverBenchmarkBluePrintType&gt;ALL_CONSTRUCTION_HEURISTIC_TYPES&lt;/solverBenchmarkBluePrintType&gt;
+    &lt;solverBenchmarkBluePrintType&gt;EVERY_CONSTRUCTION_HEURISTIC_TYPE_WITH_EVERY_LOCAL_SEARCH_TYPE&lt;/solverBenchmarkBluePrintType&gt;
   &lt;/solverBenchmarkBluePrint&gt;
 &lt;/plannerBenchmark&gt;</programlisting>
 
       <para>The following <literal>SolverBenchmarkBluePrintType</literal>s are supported:</para>
 
       <itemizedlist>
         <listitem>
-          <para><literal>ALL_CONSTRUCTION_HEURISTIC_TYPES</literal>: Run all Construction Heuristic types (First Fit,
+          <para><literal>EVERY_CONSTRUCTION_HEURISTIC_TYPE</literal>: Run every Construction Heuristic type (First Fit,
           First Fit Decreasing, Cheapest Insertion, ...).</para>
         </listitem>
       </itemizedlist>
+
+      <itemizedlist>
+        <listitem>
+          <para><literal>EVERY_LOCAL_SEARCH_TYPE</literal>: Run every Local Search type (Tabu Search, Late Acceptance,
+          ...) with the default Construction Heuristic.</para>
+        </listitem>
+      </itemizedlist>
+
+      <itemizedlist>
+        <listitem>
+          <para><literal>EVERY_CONSTRUCTION_HEURISTIC_TYPE_WITH_EVERY_LOCAL_SEARCH_TYPE</literal>: Run every
+          Construction Heuristic type with every Local Search type.</para>
+        </listitem>
+      </itemizedlist>
     </section>
 
     <section xml:id="writeTheOutputSolutionOfBenchmarkRuns">
@@ -299,7 +316,7 @@
       <para>The best solution of each benchmark run can be written in the <literal>benchmarkDirectory</literal>. By
       default, this is disabled, because the files are rarely used and considered bloat. Also, on large datasets,
       writing the best solution of each single benchmark can take quite some time and memory (causing an
-      <literal>OutOfMemoryError</literal>), especially in a verbose format like XStream.</para>
+      <literal>OutOfMemoryError</literal>), especially in a verbose format like XStream XML.</para>
 
       <para>To write those solutions in the <literal>benchmarkDirectory</literal>, enable
       <literal>writeOutputSolutionEnabled</literal>:</para>
@@ -478,7 +495,7 @@
 
     <section xml:id="benchmarkReportWorstScoreDifferencePercentageSummary">
       <title>Worst Score Difference Percentage (ROI) Summary (Graph and Table)</title>
-      
+
       <para>Shows the return on investment (ROI) per <literal>inputSolutionFile</literal> for each solver configuration
       if you'd upgrade from the worst solver configuration for that particular
       <literal>inputSolutionFile</literal>.</para>
@@ -860,7 +877,7 @@
     &lt;/problemBenchmarks&gt;</programlisting>
 
       <figure>
-       <title>Picked Move Type Step Score Diff Over Time Statistic</title>
+        <title>Picked Move Type Step Score Diff Over Time Statistic</title>
 
         <mediaobject>
           <imageobject>