PLANNER-391: Fixed ranking of uninitialized solutions

optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/SingleBenchmarkRunner.java

@@ -89,7 +89,7 @@ public SingleBenchmarkRunner call() {
                 solutionDescriptor.getVariableCount(outputSolution),
                 solutionDescriptor.getProblemScale(outputSolution));
         singleBenchmarkResult.setScore(outputSolution.getScore());
-        singleBenchmarkResult.setInitialized(solverScope.isBestSolutionInitialized());
+        singleBenchmarkResult.setUninitializedVariableCount(solverScope.getBestUninitializedVariableCount());
         singleBenchmarkResult.setTimeMillisSpent(timeMillisSpent);
         singleBenchmarkResult.setCalculateCount(solverScope.getCalculateCount());
 

optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/aggregator/swingui/BenchmarkAggregatorFrame.java

@@ -484,8 +484,8 @@ private MixedCheckBox createSolverBenchmarkCheckBox(SolverBenchmarkResult solver
                 + "Total score: %s%n"
                 + "Average time spent: %s%n"
                 + "Total winning score difference: %s",
-                solverBenchmarkResult.getAverageScore(),
-                solverBenchmarkResult.getTotalScore(),
+                solverBenchmarkResult.getAverageScoreWithUninitializedPrefix(),
+                solverBenchmarkResult.getTotalScoreWithUninitializedPrefix(),
                 solverBenchmarkResult.getAverageTimeMillisSpent() == null
                         ? "" : millisecondsSpentNumberFormat.format(solverBenchmarkResult.getAverageTimeMillisSpent()),
                 solverBenchmarkResult.getTotalWinningScoreDifference());

optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/ranking/SingleBenchmarkRankingComparator.java

@@ -21,13 +21,18 @@
 
 import org.apache.commons.lang3.builder.CompareToBuilder;
 import org.optaplanner.benchmark.impl.result.SingleBenchmarkResult;
+import org.optaplanner.core.api.score.Score;
 
 public class SingleBenchmarkRankingComparator implements Comparator<SingleBenchmarkResult>, Serializable {
 
+    private final Comparator<Score> resilientScoreComparator = new ResilientScoreComparator();
+
+    @Override
     public int compare(SingleBenchmarkResult a, SingleBenchmarkResult b) {
         return new CompareToBuilder()
-                .append(a.isFailure(), b.isFailure())
-                .append(a.getScore(), b.getScore())
+                .append(b.isFailure(), a.isFailure()) // Reverse, less is better (redundant: failed benchmarks don't get ranked at all)
+                .append(b.getUninitializedVariableCount(), a.getUninitializedVariableCount()) // Reverse, less is better
+                .append(a.getScore(), b.getScore(), resilientScoreComparator)
                 .toComparison();
     }
 

optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/ranking/TotalRankSolverRankingWeightFactory.java

@@ -39,51 +39,56 @@
  */
 public class TotalRankSolverRankingWeightFactory implements SolverRankingWeightFactory {
 
+    private final Comparator<SingleBenchmarkResult> singleBenchmarkRankingComparator = new SingleBenchmarkRankingComparator();
+
     public Comparable createRankingWeight(List<SolverBenchmarkResult> solverBenchmarkResultList, SolverBenchmarkResult solverBenchmarkResult) {
         int betterCount = 0;
         int equalCount = 0;
-        List<Score> scoreList = solverBenchmarkResult.getScoreList();
-        for (SolverBenchmarkResult otherSolverBenchmarkResult : solverBenchmarkResultList) {
-            if (otherSolverBenchmarkResult != solverBenchmarkResult) {
-                List<Score> otherScoreList = otherSolverBenchmarkResult.getScoreList();
-                // TODO the scoreList.size() can differ between SolverBenchmarks
-                for (int i = 0; i < scoreList.size(); i++) {
-                    Score score = scoreList.get(i);
-                    Score otherScore = otherScoreList.get(i);
-                    int scoreComparison = score.compareTo(otherScore);
-                    if (scoreComparison > 0) {
-                        betterCount++;
-                    } else if (scoreComparison == 0) {
-                        equalCount++;
-                    }
+        int lowerCount = 0;
+        List<SingleBenchmarkResult> singleBenchmarkResultList = solverBenchmarkResult.getSingleBenchmarkResultList();
+        for (SingleBenchmarkResult single : solverBenchmarkResult.getSingleBenchmarkResultList()) {
+            for (SingleBenchmarkResult otherSingle : single.getProblemBenchmarkResult().getSingleBenchmarkResultList()) {
+                if (single == otherSingle) {
+                    continue;
+                }
+                int scoreComparison = singleBenchmarkRankingComparator.compare(single, otherSingle);
+                if (scoreComparison > 0) {
+                    betterCount++;
+                } else if (scoreComparison == 0) {
+                    equalCount++;
+                } else {
+                    lowerCount++;
                 }
             }
         }
         return new TotalRankSolverRankingWeight(solverBenchmarkResult, betterCount, equalCount);
     }
 
-    public static class TotalRankSolverRankingWeight
-            implements Comparable<TotalRankSolverRankingWeight> {
+    public static class TotalRankSolverRankingWeight implements Comparable<TotalRankSolverRankingWeight> {
 
         private final Comparator<SolverBenchmarkResult> totalScoreSolverRankingComparator
                 = new TotalScoreSolverRankingComparator();
 
         private SolverBenchmarkResult solverBenchmarkResult;
         private int betterCount;
         private int equalCount;
+        private int lowerCount;
 
         public TotalRankSolverRankingWeight(SolverBenchmarkResult solverBenchmarkResult,
                 int betterCount, int equalCount) {
             this.solverBenchmarkResult = solverBenchmarkResult;
             this.betterCount = betterCount;
             this.equalCount = equalCount;
+            this.lowerCount = lowerCount;
         }
 
+        @Override
         public int compareTo(TotalRankSolverRankingWeight other) {
             return new CompareToBuilder()
                     .append(betterCount, other.betterCount)
                     .append(equalCount, other.equalCount)
-                    .append(solverBenchmarkResult, other.solverBenchmarkResult, totalScoreSolverRankingComparator)
+                    .append(lowerCount, other.lowerCount)
+                    .append(solverBenchmarkResult, other.solverBenchmarkResult, totalScoreSolverRankingComparator) // Tie-breaker
                     .toComparison();
         }
 
@@ -96,6 +101,7 @@ public boolean equals(Object o) {
                 return new EqualsBuilder()
                         .append(betterCount, other.betterCount)
                         .append(equalCount, other.equalCount)
+                        .append(lowerCount, other.lowerCount)
                         .appendSuper(totalScoreSolverRankingComparator
                                 .compare(solverBenchmarkResult, other.solverBenchmarkResult) == 0)
                         .isEquals();
@@ -109,6 +115,7 @@ public int hashCode() {
             return new HashCodeBuilder()
                     .append(betterCount)
                     .append(equalCount)
+                    .append(lowerCount)
                     .toHashCode();
         }
 

optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/ranking/TotalScoreSolverRankingComparator.java

@@ -35,15 +35,17 @@
  */
 public class TotalScoreSolverRankingComparator implements Comparator<SolverBenchmarkResult>, Serializable {
 
-    private final ResilientScoreComparator resilientScoreComparator
-            = new ResilientScoreComparator();
-    private final WorstScoreSolverRankingComparator worstScoreSolverRankingComparator
+    private final Comparator<Score> resilientScoreComparator = new ResilientScoreComparator();
+    private final Comparator<SolverBenchmarkResult> worstScoreSolverRankingComparator
             = new WorstScoreSolverRankingComparator();
 
+    @Override
     public int compare(SolverBenchmarkResult a, SolverBenchmarkResult b) {
         return new CompareToBuilder()
+                .append(b.getFailureCount(), a.getFailureCount()) // Reverse, less is better (redundant: failed benchmarks don't get ranked at all)
+                .append(b.getUninitializedVariableCountSum(), a.getUninitializedVariableCountSum()) // Reverse, less is better
                 .append(a.getTotalScore(), b.getTotalScore(), resilientScoreComparator)
-                .append(a, b, worstScoreSolverRankingComparator)
+                .append(a, b, worstScoreSolverRankingComparator) // Tie breaker
                 .toComparison();
     }
 

optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/ranking/WorstScoreSolverRankingComparator.java

@@ -21,27 +21,29 @@
 import java.util.Comparator;
 import java.util.List;
 
+import org.optaplanner.benchmark.impl.result.SingleBenchmarkResult;
 import org.optaplanner.benchmark.impl.result.SolverBenchmarkResult;
-import org.optaplanner.core.api.score.Score;
 
 /**
  * This ranking {@link Comparator} orders a {@link SolverBenchmarkResult} by its worst {@link Score}.
  * It minimizes the worst case scenario.
  */
 public class WorstScoreSolverRankingComparator implements Comparator<SolverBenchmarkResult>, Serializable {
 
-    private final ResilientScoreComparator resilientScoreComparator
-            = new ResilientScoreComparator();
+    private final Comparator<SingleBenchmarkResult> singleBenchmarkComparator = new SingleBenchmarkRankingComparator();
 
+    @Override
     public int compare(SolverBenchmarkResult a, SolverBenchmarkResult b) {
-        List<Score> aScoreList = a.getScoreList();
-        Collections.sort(aScoreList); // Worst scores become first in the list
-        List<Score> bScoreList = b.getScoreList();
-        Collections.sort(bScoreList); // Worst scores become first in the list
-        int aSize = aScoreList.size();
-        int bSize = bScoreList.size();
+        List<SingleBenchmarkResult> aSingleBenchmarkResultList = a.getSingleBenchmarkResultList();
+        List<SingleBenchmarkResult> bSingleBenchmarkResultList = b.getSingleBenchmarkResultList();
+        // Order scores from worst to best
+        Collections.sort(aSingleBenchmarkResultList, singleBenchmarkComparator);
+        Collections.sort(bSingleBenchmarkResultList, singleBenchmarkComparator);
+        int aSize = aSingleBenchmarkResultList.size();
+        int bSize = bSingleBenchmarkResultList.size();
         for (int i = 0; i < aSize && i < bSize; i++) {
-            int comparison = resilientScoreComparator.compare(aScoreList.get(i), bScoreList.get(i));
+            int comparison = singleBenchmarkComparator.compare(aSingleBenchmarkResultList.get(i),
+                    bSingleBenchmarkResultList.get(i));
             if (comparison != 0) {
                 return comparison;
             }

optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/result/SingleBenchmarkResult.java

@@ -34,6 +34,7 @@
 import org.optaplanner.core.api.score.FeasibilityScore;
 import org.optaplanner.core.api.score.Score;
 import org.optaplanner.core.api.solver.Solver;
+import org.optaplanner.core.impl.score.ScoreUtils;
 
 /**
  * Represents 1 benchmark for 1 {@link Solver} configuration for 1 problem instance (data set).
@@ -55,8 +56,8 @@ public class SingleBenchmarkResult {
     private Long usedMemoryAfterInputSolution = null;
 
     private Boolean succeeded = null;
-    private Boolean initialized = null;
     private Score score = null;
+    private Integer uninitializedVariableCount = null;
     private long timeMillisSpent = -1L;
     private long calculateCount = -1L;
 
@@ -135,14 +136,6 @@ public void setSucceeded(Boolean succeeded) {
         this.succeeded = succeeded;
     }
 
-    public Boolean getInitialized() {
-        return initialized;
-    }
-
-    public void setInitialized(Boolean initialized) {
-        this.initialized = initialized;
-    }
-
     public Score getScore() {
         return score;
     }
@@ -151,6 +144,14 @@ public void setScore(Score score) {
         this.score = score;
     }
 
+    public Integer getUninitializedVariableCount() {
+        return uninitializedVariableCount;
+    }
+
+    public void setUninitializedVariableCount(Integer uninitializedVariableCount) {
+        this.uninitializedVariableCount = uninitializedVariableCount;
+    }
+
     public long getTimeMillisSpent() {
         return timeMillisSpent;
     }
@@ -211,7 +212,7 @@ public boolean isSuccess() {
     }
 
     public boolean isInitialized() {
-        return initialized != null && initialized.booleanValue();
+        return uninitializedVariableCount != null && uninitializedVariableCount == 0;
     }
 
     public boolean isFailure() {
@@ -243,6 +244,10 @@ public SingleStatistic getSingleStatistic(StatisticType statisticType) {
         return effectiveSingleStatisticMap.get(statisticType);
     }
 
+    public String getScoreWithUninitializedPrefix() {
+        return ScoreUtils.getScoreWithUninitializedPrefix(uninitializedVariableCount, score);
+    }
+
     // ************************************************************************
     // Accumulate methods
     // ************************************************************************
@@ -285,8 +290,8 @@ protected static SingleBenchmarkResult createMerge(SolverBenchmarkResult solverB
         // Skip oldResult.reportDirectory
         // Skip oldResult.usedMemoryAfterInputSolution
         newResult.succeeded = oldResult.succeeded;
-        newResult.initialized = oldResult.initialized;
         newResult.score = oldResult.score;
+        newResult.uninitializedVariableCount = oldResult.uninitializedVariableCount;
         newResult.timeMillisSpent = oldResult.timeMillisSpent;
         newResult.calculateCount = oldResult.calculateCount;
 

optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/impl/result/SolverBenchmarkResult.java

@@ -58,6 +58,7 @@ public class SolverBenchmarkResult {
 
     private Integer failureCount = null;
     private Integer uninitializedSolutionCount = null;
+    private Integer uninitializedVariableCountSum = null;
     private Integer infeasibleScoreCount = null;
     private Score totalScore = null;
     private Score averageScore = null;
@@ -121,6 +122,10 @@ public Integer getUninitializedSolutionCount() {
         return uninitializedSolutionCount;
     }
 
+    public Integer getUninitializedVariableCountSum() {
+        return uninitializedVariableCountSum;
+    }
+
     public Integer getInfeasibleScoreCount() {
         return infeasibleScoreCount;
     }
@@ -257,6 +262,14 @@ public String getSolverConfigAsHtmlEscapedXml() {
         return StringEscapeUtils.ESCAPE_HTML4.translate(xml);
     }
 
+    public String getTotalScoreWithUninitializedPrefix() {
+        return ScoreUtils.getScoreWithUninitializedPrefix(uninitializedVariableCountSum, totalScore);
+    }
+
+    public String getAverageScoreWithUninitializedPrefix() {
+        return ScoreUtils.getScoreWithUninitializedPrefix(uninitializedVariableCountSum / getSingleBenchmarkResultList().size(), averageScore);
+    }
+
     // ************************************************************************
     // Accumulate methods
     // ************************************************************************
@@ -281,13 +294,15 @@ protected void determineTotalsAndAverages() {
         long totalAverageCalculateCountPerSecond = 0L;
         long totalTimeMillisSpent = 0L;
         uninitializedSolutionCount = 0;
+        uninitializedVariableCountSum = 0;
         infeasibleScoreCount = 0;
         for (SingleBenchmarkResult singleBenchmarkResult : singleBenchmarkResultList) {
             if (singleBenchmarkResult.isFailure()) {
                 failureCount++;
             } else {
                 if (!singleBenchmarkResult.isInitialized()) {
                     uninitializedSolutionCount++;
+                    uninitializedVariableCountSum += singleBenchmarkResult.getUninitializedVariableCount();
                 } else if (!singleBenchmarkResult.isScoreFeasible()) {
                     infeasibleScoreCount++;
                 }

optaplanner-benchmark/src/main/resources/org/optaplanner/benchmark/impl/report/benchmarkReport.html.ftl

@@ -174,8 +174,8 @@
                                 <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                                     <tr<#if solverBenchmarkResult.favorite> class="favoriteSolverBenchmark"</#if>>
                                         <th>${solverBenchmarkResult.name}&nbsp;<@addSolverBenchmarkBadges solverBenchmarkResult=solverBenchmarkResult/></th>
-                                        <td>${solverBenchmarkResult.totalScore!""}</td>
-                                        <td>${solverBenchmarkResult.averageScore!""}</td>
+                                        <td>${solverBenchmarkResult.totalScoreWithUninitializedPrefix!""}</td>
+                                        <td>${solverBenchmarkResult.averageScoreWithUninitializedPrefix!""}</td>
                                         <td>${solverBenchmarkResult.standardDeviationString!""}</td>
                                         <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                             <#if !solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)??>
@@ -185,7 +185,7 @@
                                                 <#if !singleBenchmarkResult.success>
                                                     <td><span class="label label-important">Failed</span></td>
                                                 <#else>
-                                                    <td>${singleBenchmarkResult.score}&nbsp;<@addSingleBenchmarkBadges singleBenchmarkResult=singleBenchmarkResult/></td>
+                                                    <td>${singleBenchmarkResult.scoreWithUninitializedPrefix}&nbsp;<@addSingleBenchmarkBadges singleBenchmarkResult=singleBenchmarkResult/></td>
                                                 </#if>
                                             </#if>
                                         </#list>

optaplanner-core/src/main/java/org/optaplanner/core/impl/score/ScoreUtils.java

@@ -79,6 +79,13 @@ public static double calculateTimeGradient(Number[] totalDiffNumbers, Number[] s
         return timeGradient;
     }
 
+    public static String getScoreWithUninitializedPrefix(int uninitializedVariableCount, Score score) {
+        if (score == null) {
+            return null; // return null, not String "null" so we can use a default value
+        }
+        return uninitializedVariableCount == 0 ? score.toString() : uninitializedVariableCount + "uninitialized/" + score.toString();
+    }
+
     private ScoreUtils() {
     }
 

optaplanner-core/src/main/java/org/optaplanner/core/impl/solver/scope/DefaultSolverScope.java

@@ -23,6 +23,7 @@
 import org.optaplanner.core.api.domain.solution.cloner.SolutionCloner;
 import org.optaplanner.core.api.score.Score;
 import org.optaplanner.core.impl.domain.solution.descriptor.SolutionDescriptor;
+import org.optaplanner.core.impl.score.ScoreUtils;
 import org.optaplanner.core.impl.score.definition.ScoreDefinition;
 import org.optaplanner.core.impl.score.director.InnerScoreDirector;
 import org.slf4j.Logger;
@@ -198,7 +199,7 @@ public void setWorkingSolutionFromBestSolution() {
     }
 
     public String getBestScoreWithUninitializedPrefix() {
-        return isBestSolutionInitialized() ? bestScore.toString() : "uninitialized/" + bestScore;
+        return ScoreUtils.getScoreWithUninitializedPrefix(bestUninitializedVariableCount, bestScore);
     }
 
 }