Skip to content
Permalink
Browse files
GEODE-9177: Adding a new analyzer that dumps data in a tabular format (
…#148)

This analyzer reads all of the benchmark results and prints one row for each
result. It does not do a comparison with a baseline.  For example:

dump_results.sh resultDir1 resultDir2 resultDir3
> Task :harness:dumpRun
Directory                                Benchmark                      ops/sec          avg latency(ms)  99%% latency(ms)
partition-10-1-thread-1K-values          PartitionBenchmark             52143.62         0.0763           0.1348
partition-10-16-thread-1K-value          PartitionBenchmark             232304.35        0.2747           0.6400
partition-10-4-thread-1K-values          PartitionBenchmark             134713.40        0.1182           0.2440
  • Loading branch information
upthewaterspout committed Apr 21, 2021
1 parent e0b1bab commit ec0a2252ab9c25db242ab438856a673662932287
Showing 7 changed files with 201 additions and 7 deletions.
@@ -35,6 +35,12 @@ repositories {
mavenCentral()
}

task(dumpRun, dependsOn: 'classes', type: JavaExec) {
main = 'org.apache.geode.perftest.analysis.DumpResults'
workingDir = project.findProperty("dumpWorkingDir") ?: rootDir
classpath = sourceSets.main.runtimeClasspath
}

task(analyzeRun, dependsOn: 'classes', type: JavaExec) {
main = 'org.apache.geode.perftest.analysis.Analyzer'
workingDir = rootDir
@@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.geode.perftest.analysis;

import static org.apache.geode.perftest.analysis.BenchmarkRunAnalyzer.getTestResult;
import static org.apache.geode.perftest.analysis.BenchmarkRunAnalyzer.getYardstickOutputForBenchmarkDir;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;

public class BenchmarkReader {
private final List<ProbeResultParser> probes = new ArrayList<>();

/**
* Add a probe to produce a comparison for. The probe expects to find output files
* in the result directory for each node of each benchmark.
*/
public void addProbe(ProbeResultParser probeResultParser) {
probes.add(probeResultParser);
}

public Map<String, Map<String, ProbeResultParser.ResultData>> readBenchmarks(File benchmarkDir)
throws IOException {

if (benchmarkDir == null || !benchmarkDir.exists()) {
throw new IllegalStateException("Could not find directory " + benchmarkDir);
}

benchmarkDir = findBenchmarkDir(benchmarkDir);

Map<String, Map<String, ProbeResultParser.ResultData>> results = new HashMap<>();
List<File> benchmarkDirs = Arrays.asList(benchmarkDir.listFiles());
benchmarkDirs.sort(File::compareTo);
for (File testDir : benchmarkDirs) {
final List<File> yardstickDirs = getYardstickOutputForBenchmarkDir(testDir);
if (yardstickDirs.isEmpty()) {
continue;
}

for (ProbeResultParser probe : probes) {
List<ProbeResultParser.ResultData> testResults = getTestResult(yardstickDirs, probe);
Map<String, ProbeResultParser.ResultData> testResultMap = testResults.stream()
.collect(Collectors.toMap(ProbeResultParser.ResultData::getDescription,
Function.identity()));
results.put(testDir.getName(), testResultMap);
}
}

return results;
}

private File findBenchmarkDir(File baseDir) {
if (!baseDir.getName().startsWith("benchmarks_")) {
Optional<String> benchmarkSubDir = Arrays.asList(baseDir.list()).stream()
.filter(subDir -> subDir.startsWith("benchmarks_")).findFirst();
String subDirName = benchmarkSubDir.orElseThrow(
() -> new IllegalStateException("No benchmark directory found in " + baseDir));
return new File(baseDir, subDirName);
} else {
return baseDir;
}
}

}
@@ -93,7 +93,7 @@ public BenchmarkRunResult analyzeTestRun(File baselineResultDir, File testResult
return result;
}

private List<ProbeResultParser.ResultData> getTestResult(List<File> resultDirs,
public static List<ProbeResultParser.ResultData> getTestResult(List<File> resultDirs,
ProbeResultParser probe) throws IOException {
probe.reset();
for (File outputDirectory : resultDirs) {
@@ -102,7 +102,7 @@ private List<ProbeResultParser.ResultData> getTestResult(List<File> resultDirs,
return probe.getProbeResults();
}

private List<File> getYardstickOutputForBenchmarkDir(File benchmarkDir) throws IOException {
public static List<File> getYardstickOutputForBenchmarkDir(File benchmarkDir) throws IOException {
return Files.walk(benchmarkDir.toPath(), 2)
.filter(path -> path.toString().endsWith(YardstickTask.YARDSTICK_OUTPUT))
.map(Path::toFile)
@@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.geode.perftest.analysis;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;

import org.apache.geode.perftest.yardstick.analysis.YardstickHdrHistogramParser;

/**
* Java main method that prints benchmark results for multiple benchmark directories
* to the terminal.
*/
public class DumpResults {

public static void main(String[] args) throws IOException {
if (args.length == 0) {
System.out.println(
"Usage: dump_results.sh benchmarkDir1 benchmarkDir2 ....");
System.exit(1);
return;
}

BenchmarkReader reader = new BenchmarkReader();
reader.addProbe(new YardstickHdrHistogramParser());
System.out.printf("%-40s %-30s %-16s %-16s %-16s\n", "Directory", "Benchmark", "ops/sec",
"avg latency(ms)", "99%% latency(ms)");

Arrays.sort(args);
for (String directoryName : args) {
final File benchmarkDir = new File(directoryName);
Map<String, Map<String, ProbeResultParser.ResultData>> directoryResults =
reader.readBenchmarks(benchmarkDir);
for (Map.Entry<String, Map<String, ProbeResultParser.ResultData>> benchmarkResult : directoryResults
.entrySet()) {

String name = benchmarkResult.getKey().replaceAll(".*\\.", "");
double opsPerSec =
benchmarkResult.getValue().get(YardstickHdrHistogramParser.AVERAGE_OPS_SECOND).value;
double latency =
benchmarkResult.getValue().get(YardstickHdrHistogramParser.AVERAGE_LATENCY).value
/ 1_000_000.0;
double latency_99 =
benchmarkResult.getValue().get(YardstickHdrHistogramParser.PERCENTILE_LATENCY_99).value
/ 1_000_000.0;

System.out.printf("%-40s %-30s %-16.2f %-16.4f %-16.4f\n", benchmarkDir.getName(), name,
opsPerSec, latency, latency_99);
}


}
}
}
@@ -32,12 +32,20 @@ public interface ProbeResultParser {
List<ResultData> getProbeResults();

class ResultData {
public String description;
public double value;
public final String description;
public final double value;

public ResultData(String description, double value) {
this.description = description;
this.value = value;
}

public String getDescription() {
return description;
}

public double getValue() {
return value;
}
}
}
@@ -33,6 +33,9 @@
*/
public class YardstickHdrHistogramParser implements ProbeResultParser {
public static final String sensorOutputFile = HdrHistogramWriter.FILE_NAME;
public static final String AVERAGE_OPS_SECOND = "average ops/second";
public static final String AVERAGE_LATENCY = "average latency";
public static final String PERCENTILE_LATENCY_99 = "99th percentile latency";

public Histogram histogram = null;

@@ -60,13 +63,13 @@ public List<ResultData> getProbeResults() {
List<ResultData> results = new ArrayList<>(3);
results.add(new ResultData("median latency", histogram.getValueAtPercentile(50)));
results.add(new ResultData("90th percentile latency", histogram.getValueAtPercentile(90)));
results.add(new ResultData("99th percentile latency", histogram.getValueAtPercentile(99)));
results.add(new ResultData(PERCENTILE_LATENCY_99, histogram.getValueAtPercentile(99)));
results.add(new ResultData("99.9th percentile latency", histogram.getValueAtPercentile(99.9)));
results.add(new ResultData("average latency", histogram.getMean()));
results.add(new ResultData(AVERAGE_LATENCY, histogram.getMean()));
results.add(new ResultData("latency standard deviation", histogram.getStdDeviation()));
results.add(new ResultData("latency standard error",
histogram.getStdDeviation() / Math.sqrt(histogram.getTotalCount())));
results.add(new ResultData("average ops/second", ((double) histogram.getTotalCount())
results.add(new ResultData(AVERAGE_OPS_SECOND, ((double) histogram.getTotalCount())
/ (histogram.getEndTimeStamp() - histogram.getStartTimeStamp()) * 1000));
return results;
}
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
set -e
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

TOP_DIR=$(git rev-parse --show-toplevel)

WORKING_DIR=$(pwd)
pushd ${TOP_DIR}
./gradlew dumpRun -PdumpWorkingDir="${WORKING_DIR}" --args "$*"
popd

0 comments on commit ec0a225

Please sign in to comment.