Skip to content
Permalink
Browse files
Automatically choose the new high watermark commit (#86)
Prints potential new high watermark
  • Loading branch information
kamilla1201 authored and pivotal-jbarrett committed Jun 25, 2019
1 parent 8ee636f commit ac1c7786f6fa8eae7933b4b587c04587243adffa
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 10 deletions.
@@ -25,6 +25,7 @@ version '1.0-SNAPSHOT'
sourceCompatibility = 1.8

def geodeVersion = project.hasProperty('geodeVersion') ? project.findProperty('geodeVersion') : '1.+'
def isCI = project.hasProperty('ci') ? project.findProperty('ci') : 0

repositories {
mavenLocal()
@@ -35,6 +36,7 @@ task(analyzeRun, dependsOn: 'classes', type: JavaExec) {
main = 'org.apache.geode.perftest.analysis.Analyzer'
workingDir = rootDir
classpath = sourceSets.main.runtimeClasspath
systemProperty 'TEST_CI', isCI
}

dependencies {
@@ -52,6 +52,8 @@ public static void main(String[] args) throws IOException {
return;
}

boolean isCI = System.getProperty("TEST_CI").equals("1");

System.out.println("Running analyzer");
System.out.println(
"Comparing test result at " + testResultArg + " to baseline at " + baselineResultArg);
@@ -66,21 +68,33 @@ public static void main(String[] args) throws IOException {
benchmarkRunResult.writeResult(new PrintWriter(System.out));
/* throw exc if failed? */

StringBuilder message = new StringBuilder();
boolean isSignificantlyBetter = false;
boolean isHighWaterCandidate = true;
StringBuilder errorMessage = new StringBuilder();
for (BenchmarkRunResult.BenchmarkResult benchmarkResult : benchmarkRunResult
.getBenchmarkResults()) {
for (BenchmarkRunResult.ProbeResult probeResult : benchmarkResult.probeResults) {
if (probeResult.description.equals("average latency")) {
if (probeResult.getDifference() >= 0.05) {
message.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" average latency is 5% worse than baseline.\n");
if (probeResult.getDifference() > 0) {
isHighWaterCandidate = false;
if (probeResult.getDifference() >= 0.05) {
errorMessage.append("BENCHMARK FAILED: ").append(benchmarkResult.name)
.append(" average latency is 5% worse than baseline.\n");
}
} else if (probeResult.getDifference() <= -0.5) {
isSignificantlyBetter = true;
}
}
}
}

if (message.length() > 0) {
System.out.println(message);
if (isCI && isHighWaterCandidate && isSignificantlyBetter) {
System.out.println(
"NEW HIGH WATERMARK COMMIT: average latency for each test is <=0.0% change from baseline AND at least one test shows a >=5% improvement in performance.\n");
}

if (errorMessage.length() > 0) {
System.out.println(errorMessage);
System.exit(1);
}

@@ -17,16 +17,49 @@ set -e
# See the License for the specific language governing permissions and
# limitations under the License.

OUTPUT_DIR=${1}

TOP_DIR=$(git rev-parse --show-toplevel)

OUTPUT_DIR=
CI=0

echo ${TOP_DIR}

while (( "$#" )); do
case "$1" in
-o|--output|--outputDir )
if [ "$2" ]; then
OUTPUT_DIR=$2
shift
fi
;;
--ci )
CI=1
;;
-h|--help|-\? )
echo "Usage: $(basename "$0") -t tag -c 4 [options ...] [-- arguments ...]"
echo "Options:"
echo "-o|--output|--outputDir : The directory containing benchmark results"
echo "--ci : Set if starting instances for Continuous Integration"
echo "-- : All subsequent arguments are passed to the benchmark task as arguments."
echo "-h|--help : This help message"
exit 1
;;
-- )
shift
break 2
;;
-?* )
printf 'Invalid option: %s\n' "$1" >&2
exit 1
;;
esac
shift
done

BASELINE_DIR="${OUTPUT_DIR}/baseline"
BRANCH_DIR="${OUTPUT_DIR}/branch"
BASELINE_BENCHMARKS="$(ls -td ${BASELINE_DIR}/benchmarks_* | tail -1)"
BRANCH_BENCHMARKS="$(ls -td ${BRANCH_DIR}/benchmarks_* | tail -1)"
pushd ${TOP_DIR}
./gradlew analyzeRun --args "${BASELINE_BENCHMARKS} ${BRANCH_BENCHMARKS}"
./gradlew analyzeRun -Pci=${CI} --args "${BASELINE_BENCHMARKS} ${BRANCH_BENCHMARKS}"
popd
@@ -39,6 +39,7 @@ BASELINE_VERSION=${DEFAULT_BASELINE_VERSION}
TAG=
METADATA=
OUTPUT=
CI=

while (( "$#" )); do
case $1 in
@@ -111,6 +112,9 @@ while (( "$#" )); do
shift
fi
;;
--ci )
CI=1
;;
-h|--help|-\? )
echo "Usage: $(basename "$0") -t tag [options ...] [-- arguments ...]"
echo "Options:"
@@ -199,4 +203,8 @@ fi

set +x

./analyze_tests.sh ${OUTPUT}
if [[ -z "${CI}" ]]; then
./analyze_tests.sh -o ${OUTPUT}
else
./analyze_tests.sh -o ${OUTPUT} --ci
fi

0 comments on commit ac1c778

Please sign in to comment.