Skip to content
Permalink
Browse files
GEODE-6340: add optional metadata to benchmark run (#50)
* GEODE-6340: add optional metadata to benchmark run

* add a metadata.json file to the test results
* get metadata from -Pmetadata in gradle command
* add cli option for metadata in run_test and run_against_benchmark
* change benchmark branch cli option from -m to -e
* update cli help message
* add default metadata values including branch names and versions

* update readme and add closing parens to help

* appease spotless
  • Loading branch information
nonbinaryprogrammer committed Feb 11, 2019
1 parent 8d181bb commit 0a29ba4762f3a79906534ffd22668405e9aacf0a
Show file tree
Hide file tree
Showing 6 changed files with 84 additions and 25 deletions.
@@ -64,6 +64,7 @@ task benchmark(type: Test) {
forkEvery 1

systemProperty 'TEST_HOSTS', project.findProperty('hosts')
systemProperty 'TEST_METADATA', project.findProperty('metadata')
systemProperty 'OUTPUT_DIR', outputDir

doFirst {
@@ -73,7 +73,6 @@ public void close() throws IOException, ExecutionException, InterruptedException
* Copy results to the provided output directory
*/
public void copyResults(File benchmarkOutput) throws IOException {
benchmarkOutput.mkdirs();
for (JVMMapping jvm : jvmMappings) {
infra.copyFromNode(jvm.getNode(), jvm.getOutputDir(),
new File(benchmarkOutput, jvm.getRole() + "-" + jvm.getId()));
@@ -18,6 +18,7 @@
package org.apache.geode.perftest.runner;

import java.io.File;
import java.io.FileWriter;
import java.util.List;
import java.util.Map;

@@ -66,7 +67,20 @@ protected void runTest(TestConfig config, String testName)
throw new IllegalStateException(
"Benchmark output directory already exists: " + benchmarkOutput.getPath());
}

benchmarkOutput.mkdirs();

String metadata = System.getProperty("TEST_METADATA");
if (!(metadata == null) && !metadata.isEmpty()) {
File metadataOutput = new File(outputDir, "metadata.json");
metadataOutput.createNewFile();
FileWriter metadataWriter = new FileWriter(metadataOutput.getAbsoluteFile(), true);

String[] metadataEntries = metadata.split(",");
for (String data : metadataEntries) {
metadataWriter.write(data + "\n");
}
metadataWriter.flush();
}

Map<String, Integer> roles = config.getRoles();
Map<String, List<String>> jvmArgs = config.getJvmArgs();
@@ -18,27 +18,58 @@ It takes two arguments. First, a tag to identify the cluster for use with other
second argument is the number of instances to create.

# run_tests.sh
Runs benchmark tests against a single branch of `geode`. Arguments are (in order)
Runs benchmark tests against a single branch of `geode` on the AWS instances with the specified tag.

* tag (the same as the cluster launched via `launch_cluster.sh`)
* branch of geode (must exist in the apache geode repository)
* branch of benchmark code to use (must exist in the apache geode-benchmarks repository)
* (Optional) output directory for results
Usage:

run_test.sh -t [tag] [-v [version] | -b [branch]] <options...>

Options:

-e : Benchmark branch (optional - defaults to develop)
-o : Output directory (optional - defaults to ./output-<date>-<tag>)
-v : Geode Version
-b : Geode Branch
-t : Cluster tag
-m : Test metadata to output to file, comma-delimited (optional)
-h : Help message

# run_against_baseline.sh
Runs benchmark tests against two branches of geode for comparison purposes. Arguments are (in order)
Runs benchmark tests against two branches of geode for comparison purposes on the AWS instances with
the specified tag.

* tag (the same as the cluster launched via `launch_cluster.sh`)
* branch of geode (must exist in the apache geode repository)
* branch of benchmark code to use (must exist in the apache geode-benchmarks repository)
Usage:

run_test.sh -t [tag] [-v [version] | -b [branch]] [-V [baseline version] | -B [baseline branch]] <options...>"

Options:

-e : Benchmark branch (optional - defaults to develop)
-o : Output directory (optional - defaults to ./output-<date>-<tag>)
-v : Geode Version
-b : Geode Branch
-V : Geode Baseline Version
-B : Geode Baseline Branch
-t : Cluster tag
-m : Test metadata to output to file, comma-delimited (optional)
-h : Help message


# destroy_cluster.sh
Destroys a cluster that you created. Arguments are the tag that you passed to launch_cluster.sh

#Example

Example 1 - run_test.sh:
```bash
./launch_cluster.sh mycluster 4
./run-tests mycluster
./destroy_cluster.sh
./run_tests.sh -t mycluster -b develop -e benchmarkBranch -m "'name':'HelenaTestingCPUs','CPU':'256','geodeBranch':'CPUTest'"
./destroy_cluster.sh mycluster
```

Example 2 - run_against_baseline.sh:
```bash
./launch_cluster.sh mycluster 4
./run-tests -t mycluster -b develop -e benchmarkBranch -m "'name':'HelenaTestingCPUs','CPU':'256','geodeBranch':'CPUTest'"
./destroy_cluster.sh mycluster
```
@@ -22,15 +22,17 @@ set -e -o pipefail
BENCHMARK_BRANCH='develop'
BRANCH='develop'

TEMP=`getopt t:b:v:B:V:m:o:h "$@"`
TEMP=`getopt t:b:v:e:B:V:m:o:h "$@"`
eval set -- "$TEMP"

while true ; do
case "$1" in
-t)
TAG=$2 ; shift 2 ;;
-m)
-e)
BENCHMARK_BRANCH=$2 ; shift 2 ;;
-m)
METADATA=$2 ; shift 2 ;;
-o)
OUTPUT=$2 ; shift 2 ;;
-b)
@@ -44,13 +46,14 @@ while true ; do
-h)
echo "Usage: run_test.sh -t [tag] [-v [version] | -b [branch]] [-V [baseline version] | -B [baseline branch]] <options...>"
echo "Options:"
echo "-m : Benchmark branch (optional - defaults to develop)"
echo "-o : Output directory (optional - defaults to ./output-<date>-<tag>"
echo "-e : Benchmark branch (optional - defaults to develop)"
echo "-o : Output directory (optional - defaults to ./output-<date>-<tag>)"
echo "-v : Geode Version"
echo "-b : Geode Branch"
echo "-V : Geode Baseline Version"
echo "-B : Geode Baseline Branch"
echo "-t : Cluster tag"
echo "-m : Test metadata to output to file, comma-delimited (optional)"
echo "-h : This help message"
shift 2
exit 1 ;;
@@ -68,6 +71,10 @@ if [ -z "${TAG}" ]; then
exit 1
fi

if [ -z "${METADATA}" ]; then
METADATA="'geode branch':'${BRANCH}','geode version':'${VERSION}','baseline branch':'${BASELINE_BRANCH}','baseline version':'${BASELINE_VERSION}','benchmark branch':'${BENCHMARK_BRANCH}'"
fi

OUTPUT=${OUTPUT:-output-${DATE}-${TAG}}

set -x
@@ -76,15 +83,15 @@ if ! [[ "$OUTPUT" = /* ]]; then
fi

if [ -z "${VERSION}" ]; then
./run_tests.sh -t ${TAG} -b ${BRANCH} -m ${BENCHMARK_BRANCH} -o ${OUTPUT}/branch
./run_tests.sh -t ${TAG} -b ${BRANCH} -e ${BENCHMARK_BRANCH} -o ${OUTPUT}/branch -m ${METADATA}
else
./run_tests.sh -t ${TAG} -v ${VERSION} -m ${BENCHMARK_BRANCH} -o ${OUTPUT}/branch
./run_tests.sh -t ${TAG} -v ${VERSION} -e ${BENCHMARK_BRANCH} -o ${OUTPUT}/branch -m ${METADATA}
fi

if [ -z "${BASELINE_VERSION}" ]; then
./run_tests.sh -t ${TAG} -b ${BASELINE_BRANCH} -m ${BENCHMARK_BRANCH} -o ${OUTPUT}/baseline
./run_tests.sh -t ${TAG} -b ${BASELINE_BRANCH} -e ${BENCHMARK_BRANCH} -o ${OUTPUT}/baseline -m ${METADATA}
else
./run_tests.sh -t ${TAG} -v ${BASELINE_VERSION} -m ${BENCHMARK_BRANCH} -o ${OUTPUT}/baseline
./run_tests.sh -t ${TAG} -v ${BASELINE_VERSION} -e ${BENCHMARK_BRANCH} -o ${OUTPUT}/baseline -m ${METADATA}
fi
set +x

@@ -21,14 +21,16 @@ set -x -e -o pipefail

BENCHMARK_BRANCH='develop'

TEMP=`getopt t:b:v:m:o:h "$@"`
TEMP=`getopt t:b:v:m:e:o:h "$@"`
eval set -- "$TEMP"

while true ; do
case "$1" in
-t)
TAG=$2 ; shift 2 ;;
-m)
METADATA=$2 ; shift 2 ;;
-e)
BENCHMARK_BRANCH=$2 ; shift 2 ;;
-o)
OUTPUT=$2 ; shift 2 ;;
@@ -39,11 +41,12 @@ while true ; do
-h)
echo "Usage: run_test.sh -t [tag] [-v [version] | -b [branch]] <options...>"
echo "Options:"
echo "-m : Benchmark branch (optional - defaults to develop)"
echo "-o : Output directory (optional - defaults to ./output-<date>-<tag>"
echo "-e : Benchmark branch (optional - defaults to develop)"
echo "-o : Output directory (optional - defaults to ./output-<date>-<tag>)"
echo "-v : Geode Version"
echo "-b : Geode Branch"
echo "-t : Cluster tag"
echo "-m : Test metadata to output to file, comma-delimited (optional)"
echo "-h : This help message"
shift 2
exit 1 ;;
@@ -109,12 +112,16 @@ if [ -z "${VERSION}" ]; then
exit 1
fi

if [ -z "${METADATA}" ]; then
METADATA="'geode branch':'${BRANCH}','geode version':'${VERSION}','benchmark branch':'${BENCHMARK_BRANCH}'"
fi


ssh ${SSH_OPTIONS} geode@$FIRST_INSTANCE "\
rm -rf geode-benchmarks && \
git clone https://github.com/apache/geode-benchmarks --branch ${BENCHMARK_BRANCH} && \
cd geode-benchmarks && \
./gradlew -PgeodeVersion=${VERSION} benchmark -Phosts=${HOSTS}"
./gradlew -PgeodeVersion=${VERSION} benchmark -Phosts=${HOSTS} -Pmetadata=${METADATA}
mkdir -p ${OUTPUT}

0 comments on commit 0a29ba4

Please sign in to comment.