Skip to content
Permalink
Browse files
GEODE-6145: Allow multiple results for a single probe analyzer (#14)
This change modifies the ProbeResultParser interface to return a list of
ResultData objects.  It takes advantage of this new functionality to return
6 results from the HDRHistogramParser (median latency, 90, 99, 99.9 percentile
latencies, average latency, and latency standard deviation).  This is probably
too noisy for the long run, but could prove useful as we evaluate different
platforms.

This change also changes the BenchmarkRunAnalyzer (invoked via gradlew
analyzeRun) to take the baseline directory as its first argument.
  • Loading branch information
Brian Rowe committed Dec 5, 2018
1 parent e146df2 commit ef2e72fec401d6a801744f7a530d9c026c9d77b0
Show file tree
Hide file tree
Showing 6 changed files with 57 additions and 37 deletions.
@@ -20,6 +20,7 @@
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;

@@ -55,7 +56,7 @@ public void addProbe(ProbeResultParser probeResultParser) {
probes.add(probeResultParser);
}

public BenchmarkRunResult analyzeTestRun(File testResultDir, File baselineResultDir)
public BenchmarkRunResult analyzeTestRun(File baselineResultDir, File testResultDir)
throws IOException {
List<File> benchmarkDirs = Arrays.asList(testResultDir.listFiles());
benchmarkDirs.sort(File::compareTo);
@@ -72,22 +73,33 @@ public BenchmarkRunResult analyzeTestRun(File testResultDir, File baselineResult
final BenchmarkRunResult.BenchmarkResult benchmarkResult =
result.addBenchmark(testDir.getName());
for (ProbeResultParser probe : probes) {
double testResult = getTestResult(testYardstickDirs, probe);
double baselineResult = getTestResult(baselineYardstickDirs, probe);
List<ProbeResultParser.ResultData> testResults = getTestResult(testYardstickDirs, probe);
List<ProbeResultParser.ResultData> baselineResults =
getTestResult(baselineYardstickDirs, probe);

benchmarkResult.addProbeResult(probe.getResultDescription(), baselineResult, testResult);
Iterator<ProbeResultParser.ResultData> testResultIter = testResults.iterator();
Iterator<ProbeResultParser.ResultData> baselineResultIter = baselineResults.iterator();

while (testResultIter.hasNext()) {
ProbeResultParser.ResultData testResult = testResultIter.next();
ProbeResultParser.ResultData baselineResult = baselineResultIter.next();

benchmarkResult.addProbeResult(testResult.description, baselineResult.value,
testResult.value);
}
}
}

return result;
}

private double getTestResult(List<File> resultDirs, ProbeResultParser probe) throws IOException {
private List<ProbeResultParser.ResultData> getTestResult(List<File> resultDirs,
ProbeResultParser probe) throws IOException {
probe.reset();
for (File outputDirectory : resultDirs) {
probe.parseResults(outputDirectory);
}
return probe.getProbeResult();
return probe.getProbeResults();
}

private List<File> getYardstickOutputForBenchmarkDir(File benchmarkDir) throws IOException {
@@ -16,8 +16,10 @@

import java.io.File;
import java.io.IOException;
import java.util.List;

public interface ProbeResultParser {

// Given a output directory for a benchmark, parse out the data for the desired probe. Note that
// this method may be passed several csv files for a run and is expected to appropriately
// aggregate the result of interest.
@@ -26,9 +28,16 @@ public interface ProbeResultParser {
// Reset the parser to a clean state where parseResults can be called again
void reset();

// Get a single float value summarizing the data for the probe.
double getProbeResult();
// Get the {description, value} pairs for the probe
List<ResultData> getProbeResults();

class ResultData {
public String description;
public double value;

// Get a text description of what the probe result is depicting
String getResultDescription();
public ResultData(String description, double value) {
this.description = description;
this.value = value;
}
}
}
@@ -17,6 +17,8 @@

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.HdrHistogram.Histogram;
import org.HdrHistogram.HistogramLogReader;
@@ -31,7 +33,6 @@
*/
public class YardstickHdrHistogramParser implements ProbeResultParser {
public static final String sensorOutputFile = HdrHistogramWriter.FILE_NAME;
public static final String probeResultDescription = "HDR 99th percentile latency";

public Histogram histogram = null;

@@ -55,13 +56,15 @@ public void reset() {
}

@Override
// Default probe result is the 99th percentile latency for the benchmark
public double getProbeResult() {
return histogram.getValueAtPercentile(99);
}
public List<ResultData> getProbeResults() {
List<ResultData> results = new ArrayList<>(3);
results.add(new ResultData("median latency", histogram.getValueAtPercentile(50)));
results.add(new ResultData("90th percentile latency", histogram.getValueAtPercentile(90)));
results.add(new ResultData("99th percentile latency", histogram.getValueAtPercentile(99)));
results.add(new ResultData("99.9th percentile latency", histogram.getValueAtPercentile(99.9)));
results.add(new ResultData("average latency", histogram.getMean()));
results.add(new ResultData("latency standard deviation", histogram.getStdDeviation()));

@Override
public String getResultDescription() {
return probeResultDescription;
return results;
}
}
@@ -21,6 +21,7 @@
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.yardstickframework.probes.PercentileProbe;

@@ -32,7 +33,7 @@
*/
public class YardstickPercentileSensorParser implements ProbeResultParser {
public static final String sensorOutputFile = "PercentileProbe.csv";
public static final String probeResultDescription = "99th percentile latency";
public static final String probeResultDescription = "YS 99th percentile latency";

private class SensorBucket {
public int latencyBucket;
@@ -74,17 +75,6 @@ public void reset() {
buckets = new ArrayList<>();
}

@Override
// Default probe result is the 99th percentile latency for the benchmark
public double getProbeResult() {
return getPercentile(99);
}

@Override
public String getResultDescription() {
return probeResultDescription;
}

private void normalizeBuckets() {
float totalPercentage = 0;
for (SensorBucket bucket : buckets) {
@@ -131,4 +121,12 @@ public double getPercentile(int target) {

return targetBucket.latencyBucket + bucketSize * percentileLocationInTargetBucket;
}

@Override
public List<ResultData> getProbeResults() {
List<ResultData> results = new ArrayList<>(1);
results.add(new ResultData(probeResultDescription, getPercentile(99)));

return results;
}
}
@@ -56,13 +56,11 @@ public void reset() {
}

@Override
public double getProbeResult() {
return getAverageThroughput();
}
public List<ResultData> getProbeResults() {
List<ResultData> results = new ArrayList<>(1);
results.add(new ResultData(probeResultDescription, getAverageThroughput()));

@Override
public String getResultDescription() {
return probeResultDescription;
return results;
}

public double getAverageThroughput() {
@@ -81,7 +81,7 @@ public void verifyResultHarvester() throws IOException {

StringWriter writer = new StringWriter();

BenchmarkRunResult results = harvester.analyzeTestRun(testFolder, baseFolder);
BenchmarkRunResult results = harvester.analyzeTestRun(baseFolder, testFolder);

BenchmarkRunResult expectedBenchmarkResult = new BenchmarkRunResult();
BenchmarkRunResult.BenchmarkResult resultA = expectedBenchmarkResult.addBenchmark("BenchmarkA");

0 comments on commit ef2e72f

Please sign in to comment.