Skip to content
Permalink
Browse files
GEODE-6147 - Fail benchmark task if average latency change is 5+% (#43)
* GEODE-6147 - Fail benchmark task if average latency change is 5+%

* Change ratio to difference
* Add difference calculation to ProbeResult
* Fail analyzeRun task if average latency difference is >= 5%

Authored-by: Sean Goller <sgoller@pivotal.io>
  • Loading branch information
smgoller committed Jan 17, 2019
1 parent e4a68d8 commit d98d389fa775e09e5f68b9dc9a13acdea5b130fa
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 9 deletions.
@@ -61,7 +61,28 @@ public static void main(String[] args) throws IOException {
analyzer.addProbe(new YardstickPercentileSensorParser());
analyzer.addProbe(new YardstickHdrHistogramParser());

analyzer.analyzeTestRun(baselineResultDir, testResultDir)
.writeResult(new PrintWriter(System.out));
BenchmarkRunResult benchmarkRunResult =
analyzer.analyzeTestRun(baselineResultDir, testResultDir);
benchmarkRunResult.writeResult(new PrintWriter(System.out));
/* throw exc if failed? */

StringBuilder message = new StringBuilder();
for (BenchmarkRunResult.BenchmarkResult benchmarkResult : benchmarkRunResult
.getBenchmarkResults()) {
for (BenchmarkRunResult.ProbeResult probeResult : benchmarkResult.probeResults) {
if (probeResult.description.equals("average latency")) {
if (probeResult.getDifference() >= 0.05) {
message.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" average latency is 5% worse than baseline.\n");
}
}
}
}

if (message.length() > 0) {
System.out.println(message);
System.exit(1);
}

}
}
@@ -40,7 +40,7 @@ public void writeResult(Writer output) throws IOException {
stream.print(String.format(" %30s", probeResult.description));
stream.print(String.format(" Baseline: %12.2f", probeResult.baseline));
stream.print(String.format(" Test: %12.2f", probeResult.test));
stream.print(String.format(" Ratio: %2.2f", probeResult.test / probeResult.baseline));
stream.print(String.format(" Difference: %+6.1f%%", probeResult.getDifference() * 100));
stream.println();
}
}
@@ -83,8 +83,8 @@ public int hashCode() {
}

static class BenchmarkResult implements Serializable {
private final String name;
private final List<ProbeResult> probeResults = new ArrayList<>();
final String name;
final List<ProbeResult> probeResults = new ArrayList<>();

public BenchmarkResult(String name) {
this.name = name;
@@ -114,17 +114,21 @@ public int hashCode() {
}
}

private static class ProbeResult implements Serializable {
private final String description;
private final double baseline;
private final double test;
static class ProbeResult implements Serializable {
final String description;
final double baseline;
final double test;

public ProbeResult(String description, double baseline, double test) {
this.description = description;
this.baseline = baseline;
this.test = test;
}

public double getDifference() {
return (test - baseline) / baseline;
}

@Override
public boolean equals(Object o) {
if (this == o) {

0 comments on commit d98d389

Please sign in to comment.