Skip to content
Permalink
Browse files
GEODE-6086: Adding a tool to allow command line analysis of test runs (
…#7)

This change adds the analyzeRun gradle target, which, when passed a baseline
result directory and test result directory via --args, will analyze the output
and report the variation of the test from the baseline.

This also improves the benchmark analyzer to scan the test result dir and
automatically parse out the benchmarks rather than needing them passed.
  • Loading branch information
Brian Rowe committed Nov 28, 2018
1 parent 4d98bf1 commit 30ae13c036a0eb69a395a83fdc3ad38943828c5b
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 57 deletions.
@@ -28,6 +28,11 @@ repositories {
mavenCentral()
}

task(analyzeRun, dependsOn: 'classes', type: JavaExec) {
main = 'org.apache.geode.perftest.analysis.Analyzer'
classpath = sourceSets.main.runtimeClasspath
}

dependencies {
testCompile group: 'junit', name: 'junit', version: '4.12'
compile group: 'com.hierynomus', name: 'sshj', version: '0.26.0'
@@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.geode.perftest.analysis;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;

import org.apache.geode.perftest.yardstick.analysis.YardstickPercentileSensorParser;
import org.apache.geode.perftest.yardstick.analysis.YardstickThroughputSensorParser;

public class Analyzer {

public static void main(String[] args) throws IOException {
if (args.length != 2) {
System.out.println("Analyzer takes two test output directories as arguments, test results followed by baseline run result.");
System.exit(1);
return;
}

String testResultArg = args[0];
String baselineResultArg = args[1];

File testResultDir = new File(testResultArg);
File baselineResultDir = new File(baselineResultArg);

boolean valid = true;
if (!testResultDir.exists()) {
System.out.println("Unable to find test result directory: " + testResultArg);
valid = false;
}
if (!baselineResultDir.exists()) {
System.out.println("Unable to find test result directory: " + baselineResultArg);
valid = false;
}
if (!valid) {
System.exit(1);
return;
}

System.out.println("Running analyzer");
System.out.println("Comparing test result at " + testResultArg + " to baseline at " + baselineResultArg);

BenchmarkRunAnalyzer analyzer = new BenchmarkRunAnalyzer();
analyzer.addProbe(new YardstickThroughputSensorParser());
analyzer.addProbe(new YardstickPercentileSensorParser());

analyzer.analyzeTestRun(testResultDir, baselineResultDir, new PrintWriter(System.out));
}
}
@@ -15,13 +15,15 @@
package org.apache.geode.perftest.analysis;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

import org.apache.geode.perftest.yardstick.YardstickTask;

@@ -44,17 +46,8 @@
* </pre>
*/
public class BenchmarkRunAnalyzer {
private final List<SensorData> benchmarks = new ArrayList<>();
private final List<ProbeResultParser> probes = new ArrayList<>();

/**
* Add a benchmark to be analyzed. The benchmark is expected to exist
* in both result directories passed to {@link #analyzeTestRun(File, File, Writer)}
*/
public void addBenchmark(String name, String testResultDir, List<String> nodeNames) {
benchmarks.add(new SensorData(name, testResultDir, nodeNames));
}

/**
* Add a probe to produce a comparison for. The probe expects to find output files
* in the result directory for each node of each benchmark.
@@ -65,19 +58,26 @@ public void addProbe(ProbeResultParser probeResultParser) {

public void analyzeTestRun(File testResultDir, File baselineResultDir, Writer output)
throws IOException {
List<File> benchmarkDirs = Arrays.asList(testResultDir.listFiles());

PrintWriter stream = new PrintWriter(output);
for (SensorData benchmark : benchmarks) {
stream.println("-- " + benchmark.benchmarkName + " --");
for (File testDir : benchmarkDirs) {
final List<File> yardstickOutputDirsForTest = getYardstickOutputForBenchmarkDir(testDir);
if (yardstickOutputDirsForTest.isEmpty()) {
continue;
}
File baselineDir = new File(baselineResultDir, testDir.getName());
stream.println("-- " + testDir.getName() + " --");
for (ProbeResultParser probe : probes) {
stream.println(probe.getResultDescription());
for (String node : benchmark.nodesToParse) {
probe.parseResults(getBenchmarkOutputDir(testResultDir, benchmark, node));
for (File outputDirectory : yardstickOutputDirsForTest) {
probe.parseResults(outputDirectory);
}
double testResult = probe.getProbeResult();
stream.println("Result: " + String.valueOf(testResult));
probe.reset();
for (String node : benchmark.nodesToParse) {
probe.parseResults(getBenchmarkOutputDir(baselineResultDir, benchmark, node));
for (File outputDirectory : getYardstickOutputForBenchmarkDir(baselineDir)) {
probe.parseResults(outputDirectory);
}
double baselineResult = probe.getProbeResult();
stream.println("Baseline: " + String.valueOf(baselineResult));
@@ -89,35 +89,10 @@ public void analyzeTestRun(File testResultDir, File baselineResultDir, Writer ou
stream.flush();
}

private File getBenchmarkOutputDir(File testResultDir, SensorData benchmark,
String node) {
File benchmarkDir = new File(testResultDir, benchmark.benchmarkSubdirectory);
File nodeDir = new File(benchmarkDir, node);

File[] files = nodeDir.listFiles((dir, name) -> name.contains(YardstickTask.YARDSTICK_OUTPUT));

if(files == null || files.length != 1) {
throw new IllegalStateException("Expected at least one subdirectory in " + nodeDir
+ " with the name *" +YardstickTask.YARDSTICK_OUTPUT);
}

return files[0];
private List<File> getYardstickOutputForBenchmarkDir(File benchmarkDir) throws IOException {
return Files.walk(benchmarkDir.toPath(), 2)
.filter(path -> path.toString().endsWith(YardstickTask.YARDSTICK_OUTPUT))
.map(Path::toFile)
.collect(Collectors.toList());
}

// TODO: depending on how run output is stored, this data may be excessive or insufficient
// The present assumption is each benchmark contains an arbitrarily named result directory
// containing subdirectories for each node. Those subdirectories then contain the probe output
// files for the run, for that node.
private static class SensorData {
private final String benchmarkName;
private final String benchmarkSubdirectory;
private final List<String> nodesToParse;

public SensorData(String benchmarkName, String benchmarkSubdirectory, List<String> nodesNames) {
this.benchmarkName = benchmarkName;
this.benchmarkSubdirectory = benchmarkSubdirectory;
this.nodesToParse = nodesNames;
}
}

}
@@ -23,16 +23,13 @@
import java.io.PrintStream;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;

import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import org.apache.geode.perftest.analysis.BenchmarkRunAnalyzer;
import org.apache.geode.perftest.yardstick.analysis.YardstickPercentileSensorParser;
import org.apache.geode.perftest.yardstick.analysis.YardstickThroughputSensorParser;

@@ -47,6 +44,8 @@ public void verifyResultHarvester() throws IOException {
final File testBenchmarkA2 = temporaryFolder.newFolder("testFolder","BenchmarkA","client2", "20181121-111516-yardstick-output");
final File testBenchmarkB1 = temporaryFolder.newFolder("testFolder","BenchmarkB","client1", "20181121-111516-yardstick-output");
final File testBenchmarkB2 = temporaryFolder.newFolder("testFolder","BenchmarkB","client2", "20181121-111516-yardstick-output");
temporaryFolder.newFolder(testFolder.getName(), "junkfolder");
new File(testFolder, "junkfile").createNewFile();
final File baseFolder = temporaryFolder.newFolder("baseFolder");
final File baseBenchmarkA1 = temporaryFolder.newFolder("baseFolder","BenchmarkA","client1", "20181121-111516-yardstick-output");
final File baseBenchmarkA2 = temporaryFolder.newFolder("baseFolder","BenchmarkA","client2", "20181121-111516-yardstick-output");
@@ -71,13 +70,7 @@ public void verifyResultHarvester() throws IOException {
populateThroughputCSV(baseBenchmarkB2, new double[] {10, 15, 20, 25, 30}); // Avg 20
populatePercentileCSV(baseBenchmarkB2, new double[] {0, 0, 0, 99, 1}); // 300

List<String> nodes = new ArrayList(2);
nodes.add("client1");
nodes.add("client2");

BenchmarkRunAnalyzer harvester = new BenchmarkRunAnalyzer();
harvester.addBenchmark("Alpha", "BenchmarkA", nodes);
harvester.addBenchmark("Beta", "BenchmarkB", nodes);
harvester.addProbe(new YardstickThroughputSensorParser());
harvester.addProbe(new YardstickPercentileSensorParser());

@@ -88,8 +81,8 @@ public void verifyResultHarvester() throws IOException {
System.out.println(writer.toString());
BufferedReader resultReader = new BufferedReader(new StringReader(writer.toString()));

validatedBenchmark(resultReader, "Alpha", 20, 300, 25, 200);
validatedBenchmark(resultReader, "Beta", 25, 400, 20, 400);
validatedBenchmark(resultReader, "BenchmarkA", 20, 300, 25, 200);
validatedBenchmark(resultReader, "BenchmarkB", 25, 400, 20, 400);
}

private void populateThroughputCSV(File targetDirectory, double[] perSecondThroughputs)

0 comments on commit 30ae13c

Please sign in to comment.