Skip to content

Commit

Permalink
Rename Comperators + order getters before non-simple methods + use to…
Browse files Browse the repository at this point in the history 
…tal score instead of average score
  • Loading branch information
@ge0ffrey
ge0ffrey committed Jan 18, 2017
1 parent 291706e commit 1e7bccb
Show file tree
Hide file tree
Showing 14 changed files with 77 additions and 76 deletions.
4 changes: 2 additions & 2 deletions .../SubSingleBenchmarkRankingComparator.java → ...eSubSingleBenchmarkRankingComparator.java
View file
Expand Up @@ -22,7 +22,7 @@
import org.optaplanner.benchmark.impl.result.SubSingleBenchmarkResult;
import org.optaplanner.core.api.score.Score;

public class SubSingleBenchmarkRankingComparator implements Comparator<SubSingleBenchmarkResult>, Serializable {
public class ScoreSubSingleBenchmarkRankingComparator implements Comparator<SubSingleBenchmarkResult>, Serializable {

private final Comparator<Score> resilientScoreComparator = new ResilientScoreComparator();

Expand All @@ -31,7 +31,7 @@ public int compare(SubSingleBenchmarkResult a, SubSingleBenchmarkResult b) {
return Comparator
// Reverse, less is better (redundant: failed benchmarks don't get ranked at all)
.comparing(SubSingleBenchmarkResult::hasAnyFailure, Comparator.reverseOrder())
.thenComparing(SubSingleBenchmarkResult::getAverageScore, resilientScoreComparator)
.thenComparing(SubSingleBenchmarkResult::getScore, resilientScoreComparator)
.compare(a, b);
}

Expand Down
2 changes: 1 addition & 1 deletion ...ingleBenchmarkRankingScoreComparator.java → ...ubSingleBenchmarkRankBasedComparator.java
View file
Expand Up @@ -21,7 +21,7 @@

import org.optaplanner.benchmark.impl.result.SubSingleBenchmarkResult;

public class SubSingleBenchmarkRankingScoreComparator implements Comparator<SubSingleBenchmarkResult>, Serializable {
public class SubSingleBenchmarkRankBasedComparator implements Comparator<SubSingleBenchmarkResult>, Serializable {

@Override
public int compare(SubSingleBenchmarkResult a, SubSingleBenchmarkResult b) {
Expand Down
34 changes: 17 additions & 17 deletions ...main/java/org/optaplanner/benchmark/impl/ranking/TotalRankSolverRankingWeightFactory.java
View file
Expand Up @@ -37,7 +37,7 @@
*/
public class TotalRankSolverRankingWeightFactory implements SolverRankingWeightFactory {

private final Comparator<SingleBenchmarkResult> singleBenchmarkRankingComparator = new SingleBenchmarkRankingComparator();
private final Comparator<SingleBenchmarkResult> singleBenchmarkRankingComparator = new TotalScoreSingleBenchmarkRankingComparator();

@Override
public Comparable createRankingWeight(List<SolverBenchmarkResult> solverBenchmarkResultList, SolverBenchmarkResult solverBenchmarkResult) {
Expand Down Expand Up @@ -74,6 +74,22 @@ public static class TotalRankSolverRankingWeight implements Comparable<TotalRank
private int equalCount;
private int lowerCount;

public SolverBenchmarkResult getSolverBenchmarkResult() {
return solverBenchmarkResult;
}

public int getBetterCount() {
return betterCount;
}

public int getEqualCount() {
return equalCount;
}

public int getLowerCount() {
return lowerCount;
}

public TotalRankSolverRankingWeight(SolverBenchmarkResult solverBenchmarkResult,
int betterCount, int equalCount, int lowerCount) {
this.solverBenchmarkResult = solverBenchmarkResult;
Expand Down Expand Up @@ -109,22 +125,6 @@ public int hashCode() {
return Objects.hash(getBetterCount(), getEqualCount(), getLowerCount());
}

public int getBetterCount() {
return betterCount;
}

public int getEqualCount() {
return equalCount;
}

public int getLowerCount() {
return lowerCount;
}

public SolverBenchmarkResult getSolverBenchmarkResult() {
return solverBenchmarkResult;
}

}

}
4 changes: 2 additions & 2 deletions ...ing/SingleBenchmarkRankingComparator.java → ...coreSingleBenchmarkRankingComparator.java
View file
Expand Up @@ -22,7 +22,7 @@
import org.optaplanner.benchmark.impl.result.SingleBenchmarkResult;
import org.optaplanner.core.api.score.Score;

public class SingleBenchmarkRankingComparator implements Comparator<SingleBenchmarkResult>, Serializable {
public class TotalScoreSingleBenchmarkRankingComparator implements Comparator<SingleBenchmarkResult>, Serializable {

private final Comparator<Score> resilientScoreComparator = new ResilientScoreComparator();

Expand All @@ -31,7 +31,7 @@ public int compare(SingleBenchmarkResult a, SingleBenchmarkResult b) {
return Comparator
// Reverse, less is better (redundant: failed benchmarks don't get ranked at all)
.comparing(SingleBenchmarkResult::hasAnyFailure, Comparator.reverseOrder())
.thenComparing(SingleBenchmarkResult::getAverageScore, resilientScoreComparator)
.thenComparing(SingleBenchmarkResult::getTotalScore, resilientScoreComparator)
.compare(a, b);
}

Expand Down
2 changes: 1 addition & 1 deletion ...c/main/java/org/optaplanner/benchmark/impl/ranking/WorstScoreSolverRankingComparator.java
View file
Expand Up @@ -31,7 +31,7 @@
*/
public class WorstScoreSolverRankingComparator implements Comparator<SolverBenchmarkResult>, Serializable {

private final Comparator<SingleBenchmarkResult> singleBenchmarkComparator = new SingleBenchmarkRankingComparator();
private final Comparator<SingleBenchmarkResult> singleBenchmarkComparator = new TotalScoreSingleBenchmarkRankingComparator();

@Override
public int compare(SolverBenchmarkResult a, SolverBenchmarkResult b) {
Expand Down
5 changes: 2 additions & 3 deletions ...benchmark/src/main/java/org/optaplanner/benchmark/impl/result/ProblemBenchmarkResult.java
View file
Expand Up @@ -33,12 +33,11 @@
import com.thoughtworks.xstream.annotations.XStreamAlias;
import com.thoughtworks.xstream.annotations.XStreamImplicit;
import com.thoughtworks.xstream.annotations.XStreamOmitField;
import org.apache.commons.lang3.StringUtils;
import org.optaplanner.benchmark.config.ProblemBenchmarksConfig;
import org.optaplanner.benchmark.config.statistic.ProblemStatisticType;
import org.optaplanner.benchmark.config.statistic.SingleStatisticType;
import org.optaplanner.benchmark.impl.measurement.ScoreDifferencePercentage;
import org.optaplanner.benchmark.impl.ranking.SingleBenchmarkRankingComparator;
import org.optaplanner.benchmark.impl.ranking.TotalScoreSingleBenchmarkRankingComparator;
import org.optaplanner.benchmark.impl.report.BenchmarkReport;
import org.optaplanner.benchmark.impl.report.ReportHelper;
import org.optaplanner.benchmark.impl.statistic.ProblemStatistic;
Expand Down Expand Up @@ -351,7 +350,7 @@ private void determineTotalsAndAveragesAndRanking() {
}

private void determineRanking(List<SingleBenchmarkResult> rankedSingleBenchmarkResultList) {
Comparator singleBenchmarkRankingComparator = new SingleBenchmarkRankingComparator();
Comparator singleBenchmarkRankingComparator = new TotalScoreSingleBenchmarkRankingComparator();
Collections.sort(rankedSingleBenchmarkResultList, Collections.reverseOrder(singleBenchmarkRankingComparator));
int ranking = 0;
SingleBenchmarkResult previousSingleBenchmarkResult = null;
Expand Down
14 changes: 8 additions & 6 deletions ...-benchmark/src/main/java/org/optaplanner/benchmark/impl/result/SingleBenchmarkResult.java
View file
Expand Up @@ -28,13 +28,14 @@
import com.thoughtworks.xstream.annotations.XStreamOmitField;
import org.optaplanner.benchmark.config.statistic.ProblemStatisticType;
import org.optaplanner.benchmark.impl.measurement.ScoreDifferencePercentage;
import org.optaplanner.benchmark.impl.ranking.SubSingleBenchmarkRankingComparator;
import org.optaplanner.benchmark.impl.ranking.SubSingleBenchmarkRankingScoreComparator;
import org.optaplanner.benchmark.impl.ranking.ScoreSubSingleBenchmarkRankingComparator;
import org.optaplanner.benchmark.impl.ranking.SubSingleBenchmarkRankBasedComparator;
import org.optaplanner.benchmark.impl.report.BenchmarkReport;
import org.optaplanner.benchmark.impl.statistic.StatisticUtils;
import org.optaplanner.benchmark.impl.statistic.SubSingleStatistic;
import org.optaplanner.core.api.score.FeasibilityScore;
import org.optaplanner.core.api.score.Score;
import org.optaplanner.core.api.score.buildin.simple.SimpleScore;
import org.optaplanner.core.api.solver.Solver;
import org.optaplanner.core.config.util.ConfigUtils;
import org.slf4j.Logger;
Expand Down Expand Up @@ -187,8 +188,9 @@ public Score getAverageScore() {
return averageScore;
}

public void setAverageScore(Score averageScore) {
this.averageScore = averageScore;
public void setAverageAndTotalScoreForTesting(Score averageAndTotalScore) {
this.averageScore = averageAndTotalScore;
this.totalScore = averageAndTotalScore;
}

public SubSingleBenchmarkResult getMedian() {
Expand Down Expand Up @@ -323,7 +325,7 @@ private void determineRepresentativeSubSingleBenchmarkResult() {
}
List<SubSingleBenchmarkResult> subSingleBenchmarkResultListCopy = new ArrayList<>(subSingleBenchmarkResultList);
// sort (according to ranking) so that the best subSingle is at index 0
Collections.sort(subSingleBenchmarkResultListCopy, new SubSingleBenchmarkRankingScoreComparator());
Collections.sort(subSingleBenchmarkResultListCopy, new SubSingleBenchmarkRankBasedComparator());
best = subSingleBenchmarkResultListCopy.get(0);
worst = subSingleBenchmarkResultListCopy.get(subSingleBenchmarkResultListCopy.size() - 1);
median = subSingleBenchmarkResultListCopy.get(ConfigUtils.ceilDivide(subSingleBenchmarkResultListCopy.size() - 1, 2));
Expand Down Expand Up @@ -368,7 +370,7 @@ private void determineTotalsAndAveragesAndRanking() {
}

private void determineRanking(List<SubSingleBenchmarkResult> rankedSubSingleBenchmarkResultList) {
Comparator subSingleBenchmarkRankingComparator = new SubSingleBenchmarkRankingComparator();
Comparator subSingleBenchmarkRankingComparator = new ScoreSubSingleBenchmarkRankingComparator();
Collections.sort(rankedSubSingleBenchmarkResultList, Collections.reverseOrder(subSingleBenchmarkRankingComparator));
int ranking = 0;
SubSingleBenchmarkResult previousSubSingleBenchmarkResult = null;
Expand Down
34 changes: 17 additions & 17 deletions ...test/java/org/optaplanner/benchmark/impl/ranking/AbstractSolverRankingComparatorTest.java
View file
Expand Up @@ -21,9 +21,11 @@
import org.optaplanner.benchmark.impl.measurement.ScoreDifferencePercentage;
import org.optaplanner.benchmark.impl.result.ProblemBenchmarkResult;
import org.optaplanner.benchmark.impl.result.SingleBenchmarkResult;
import org.optaplanner.core.api.score.Score;
import org.optaplanner.core.api.score.buildin.hardsoftlong.HardSoftLongScore;
import org.optaplanner.core.api.score.buildin.simple.SimpleScore;


public abstract class AbstractSolverRankingComparatorTest {

protected ProblemBenchmarkResult addProblemBenchmark(List<SingleBenchmarkResult> singleBenchmarkResultList) {
Expand All @@ -37,30 +39,28 @@ protected ProblemBenchmarkResult addProblemBenchmark(List<SingleBenchmarkResult>

protected SingleBenchmarkResult addSingleBenchmark(List<SingleBenchmarkResult> singleBenchmarkResultList,
int score, int bestScore, int worstScore) {
SingleBenchmarkResult singleBenchmarkResult = new SingleBenchmarkResult(null, null);
SimpleScore scoreObject = SimpleScore.valueOf(score);
SimpleScore bestScoreObject = SimpleScore.valueOf(bestScore);
SimpleScore worstScoreObject = SimpleScore.valueOf(worstScore);
singleBenchmarkResult.setFailureCount(0);
singleBenchmarkResult.setAverageScore(scoreObject);
singleBenchmarkResult.setWinningScoreDifference(scoreObject.subtract(bestScoreObject));
singleBenchmarkResult.setWorstScoreDifferencePercentage(
ScoreDifferencePercentage.calculateScoreDifferencePercentage(worstScoreObject, scoreObject));
singleBenchmarkResultList.add(singleBenchmarkResult);
return singleBenchmarkResult;
return addSingleBenchmark(singleBenchmarkResultList,
SimpleScore.valueOf(score),
SimpleScore.valueOf(bestScore),
SimpleScore.valueOf(worstScore));
}

protected SingleBenchmarkResult addSingleBenchmarkWithHardSoftLongScore(List<SingleBenchmarkResult> singleBenchmarkResultList,
long hardScore, long softScore, long hardBestScore, long softBestScore, long hardWorstScore, long softWorstScore) {
return addSingleBenchmark(singleBenchmarkResultList,
HardSoftLongScore.valueOf(hardScore, softScore),
HardSoftLongScore.valueOf(hardBestScore, softBestScore),
HardSoftLongScore.valueOf(hardWorstScore, softWorstScore));
}

protected <S extends Score<S>> SingleBenchmarkResult addSingleBenchmark(List<SingleBenchmarkResult> singleBenchmarkResultList,
S score, S bestScore, S worstScore) {
SingleBenchmarkResult singleBenchmarkResult = new SingleBenchmarkResult(null, null);
HardSoftLongScore scoreObject = HardSoftLongScore.valueOf(hardScore, softScore);
HardSoftLongScore bestScoreObject = HardSoftLongScore.valueOf(hardBestScore, softBestScore);
HardSoftLongScore worstScoreObject = HardSoftLongScore.valueOf(hardWorstScore, softWorstScore);
singleBenchmarkResult.setFailureCount(0);
singleBenchmarkResult.setAverageScore(scoreObject);
singleBenchmarkResult.setWinningScoreDifference(scoreObject.subtract(bestScoreObject));
singleBenchmarkResult.setAverageAndTotalScoreForTesting(score);
singleBenchmarkResult.setWinningScoreDifference(score.subtract(bestScore));
singleBenchmarkResult.setWorstScoreDifferencePercentage(
ScoreDifferencePercentage.calculateScoreDifferencePercentage(worstScoreObject, scoreObject));
ScoreDifferencePercentage.calculateScoreDifferencePercentage(worstScore, score));
singleBenchmarkResultList.add(singleBenchmarkResult);
return singleBenchmarkResult;
}
Expand Down
4 changes: 2 additions & 2 deletions ...SingleBenchmarkRankingComparatorTest.java → ...SingleBenchmarkRankingComparatorTest.java
View file
Expand Up @@ -26,11 +26,11 @@
import static org.mockito.Mockito.mock;
import static org.optaplanner.core.impl.testdata.util.PlannerAssert.assertCompareToOrder;

public class SubSingleBenchmarkRankingComparatorTest {
public class ScoreSubSingleBenchmarkRankingComparatorTest {

@Test
public void compareTo() {
SubSingleBenchmarkRankingComparator comparator = new SubSingleBenchmarkRankingComparator();
ScoreSubSingleBenchmarkRankingComparator comparator = new ScoreSubSingleBenchmarkRankingComparator();
SingleBenchmarkResult singleBenchmarkResult = new SingleBenchmarkResult(mock(SolverBenchmarkResult.class), mock(ProblemBenchmarkResult.class));
SubSingleBenchmarkResult a = new SubSingleBenchmarkResult(singleBenchmarkResult, 0);
a.setSucceeded(false);
Expand Down
12 changes: 6 additions & 6 deletions .../java/org/optaplanner/benchmark/impl/ranking/TotalRankSolverRankingWeightFactoryTest.java
View file
Expand Up @@ -130,20 +130,20 @@ public void uninitializedSingleBenchmarks() {
Comparable bWeight = factory.createRankingWeight(solverBenchmarkResultList, b);
assertCompareToEquals(aWeight, bWeight);

a0.setAverageScore(SimpleScore.valueOfUninitialized(-100, -1000));
b0.setAverageScore(SimpleScore.valueOfUninitialized(-100, -1000));
a0.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-100, -1000));
b0.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-100, -1000));
a.accumulateResults(benchmarkReport);
b.accumulateResults(benchmarkReport);
// ranks, uninitialized variable counts, total scores and worst scores are equal
assertCompareToEquals(aWeight, bWeight);

b0.setAverageScore(SimpleScore.valueOf(-1000));
b1.setAverageScore(SimpleScore.valueOfUninitialized(-100, -400));
b0.setAverageAndTotalScoreForTesting(SimpleScore.valueOf(-1000));
b1.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-100, -400));
b.accumulateResults(benchmarkReport);
// ranks, uninitialized variable counts and total scores are equal, A loses on worst score (tie-breaker)
assertCompareToOrder(aWeight, bWeight);

b1.setAverageScore(SimpleScore.valueOfUninitialized(-101, -400));
b1.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-101, -400));
b.accumulateResults(benchmarkReport);
// ranks are equal, uninitialized variable count is bigger in B
assertCompareToOrder(bWeight, aWeight);
Expand Down Expand Up @@ -250,7 +250,7 @@ public void disjunctEqualPlannerBenchmarks() {
}

@Test
public void overlappingPlannnerBenchmarks() {
public void overlappingPlannerBenchmarks() {
SingleBenchmarkResult a0 = addSingleBenchmark(aSingleBenchmarkResultList, -1000, -30, -1000);
SingleBenchmarkResult a1 = addSingleBenchmark(aSingleBenchmarkResultList, -400, -30, -1000);
SingleBenchmarkResult a2 = addSingleBenchmark(aSingleBenchmarkResultList, -30, -30, -1000);
Expand Down
14 changes: 7 additions & 7 deletions ...SingleBenchmarkRankingComparatorTest.java → ...SingleBenchmarkRankingComparatorTest.java
View file
Expand Up @@ -22,26 +22,26 @@
import org.optaplanner.benchmark.impl.result.SolverBenchmarkResult;
import org.optaplanner.core.api.score.buildin.simple.SimpleScore;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.*;
import static org.optaplanner.core.impl.testdata.util.PlannerAssert.assertCompareToOrder;

public class SingleBenchmarkRankingComparatorTest {
public class TotalScoreSingleBenchmarkRankingComparatorTest {

@Test
public void compareTo() {
SingleBenchmarkRankingComparator comparator = new SingleBenchmarkRankingComparator();
TotalScoreSingleBenchmarkRankingComparator comparator = new TotalScoreSingleBenchmarkRankingComparator();
SingleBenchmarkResult a = new SingleBenchmarkResult(mock(SolverBenchmarkResult.class), mock(ProblemBenchmarkResult.class));
a.setFailureCount(1);
a.setAverageScore(null);
a.setAverageAndTotalScoreForTesting(null);
SingleBenchmarkResult b = new SingleBenchmarkResult(mock(SolverBenchmarkResult.class), mock(ProblemBenchmarkResult.class));
b.setFailureCount(0);
b.setAverageScore(SimpleScore.valueOfUninitialized(-7, -1));
b.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-7, -1));
SingleBenchmarkResult c = new SingleBenchmarkResult(mock(SolverBenchmarkResult.class), mock(ProblemBenchmarkResult.class));
c.setFailureCount(0);
c.setAverageScore(SimpleScore.valueOf(-300));
c.setAverageAndTotalScoreForTesting(SimpleScore.valueOf(-300));
SingleBenchmarkResult d = new SingleBenchmarkResult(mock(SolverBenchmarkResult.class), mock(ProblemBenchmarkResult.class));
d.setFailureCount(0);
d.setAverageScore(SimpleScore.valueOf(-20));
d.setAverageAndTotalScoreForTesting(SimpleScore.valueOf(-20));
assertCompareToOrder(comparator, a, b, c, d);
}

Expand Down
10 changes: 5 additions & 5 deletions ...st/java/org/optaplanner/benchmark/impl/ranking/TotalScoreSolverRankingComparatorTest.java
View file
Expand Up @@ -107,20 +107,20 @@ public void uninitializedSingleBenchmarks() {
b.accumulateResults(benchmarkReport);
assertCompareToEquals(comparator, a, b);

a0.setAverageScore(SimpleScore.valueOfUninitialized(-100, -1000));
b1.setAverageScore(SimpleScore.valueOfUninitialized(-100, -400));
a0.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-100, -1000));
b1.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-100, -400));
a.accumulateResults(benchmarkReport);
b.accumulateResults(benchmarkReport);
// uninitialized variable count and total score are equal, A is worse on worst score (tie-breaker)
assertCompareToOrder(comparator, a, b);

b0.setAverageScore(SimpleScore.valueOfUninitialized(-100, -1000));
b0.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-100, -1000));
b.accumulateResults(benchmarkReport);
// uninitialized variable count is bigger in B
assertCompareToOrder(comparator, b, a);

b0.setAverageScore(SimpleScore.valueOf(-1000));
b1.setAverageScore(SimpleScore.valueOfUninitialized(-99, -400));
b0.setAverageAndTotalScoreForTesting(SimpleScore.valueOf(-1000));
b1.setAverageAndTotalScoreForTesting(SimpleScore.valueOfUninitialized(-99, -400));
b.accumulateResults(benchmarkReport);
// uninitialized variable count is bigger in A
assertCompareToOrder(comparator, a, b);
Expand Down

0 comments on commit 1e7bccb

Please sign in to comment.