From 492b8673f366d33994ab420e3b0d446e3267cd49 Mon Sep 17 00:00:00 2001 From: Abhyudaya Sharma Date: Wed, 5 Jun 2019 10:43:53 +0530 Subject: [PATCH 1/5] Add profile to run benchmarks only when the `benchmark` property is set --- pom.xml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pom.xml b/pom.xml index be0def0432..305a3ee0cf 100644 --- a/pom.xml +++ b/pom.xml @@ -1493,5 +1493,28 @@ + + + jmh-benchmark + + + benchmark + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/*Benchmark.java + **/Benchmark*.java + + + + + + From 1a892929f5d767a9e5922ef077b1d59bd079d7cb Mon Sep 17 00:00:00 2001 From: Abhyudaya Sharma Date: Wed, 5 Jun 2019 16:02:42 +0530 Subject: [PATCH 2/5] Add Integration test for the `benchmark` profile --- src/it/benchmark/invoker.properties | 1 + src/it/benchmark/pom.xml | 31 +++++++++++++++++++ src/it/benchmark/postbuild.groovy | 4 +++ .../test/java/benchmark/DontRunMeTest.java | 10 ++++++ .../test/java/benchmark/TestBenchmark.java | 16 ++++++++++ 5 files changed, 62 insertions(+) create mode 100644 src/it/benchmark/invoker.properties create mode 100644 src/it/benchmark/pom.xml create mode 100644 src/it/benchmark/postbuild.groovy create mode 100644 src/it/benchmark/src/test/java/benchmark/DontRunMeTest.java create mode 100644 src/it/benchmark/src/test/java/benchmark/TestBenchmark.java diff --git a/src/it/benchmark/invoker.properties b/src/it/benchmark/invoker.properties new file mode 100644 index 0000000000..85dbb8329f --- /dev/null +++ b/src/it/benchmark/invoker.properties @@ -0,0 +1 @@ +invoker.goals=-Dbenchmark clean test diff --git a/src/it/benchmark/pom.xml b/src/it/benchmark/pom.xml new file mode 100644 index 0000000000..e56be8c582 --- /dev/null +++ b/src/it/benchmark/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + org.jenkins-ci.plugins + plugin + @project.version@ + + + org.jenkins-ci.plugins.its + benchmark-it + 1.0-SNAPSHOT + jar + + 2.60.3 + 8 + + + + repo.jenkins-ci.org + https://repo.jenkins-ci.org/public/ + + + + + repo.jenkins-ci.org + https://repo.jenkins-ci.org/public/ + + + diff --git a/src/it/benchmark/postbuild.groovy b/src/it/benchmark/postbuild.groovy new file mode 100644 index 0000000000..dc7cf15884 --- /dev/null +++ b/src/it/benchmark/postbuild.groovy @@ -0,0 +1,4 @@ +// check if the benchmark was run +def file = new File(basedir, 'target/benchmark-run') +assert file.exists() +file.deleteDir() diff --git a/src/it/benchmark/src/test/java/benchmark/DontRunMeTest.java b/src/it/benchmark/src/test/java/benchmark/DontRunMeTest.java new file mode 100644 index 0000000000..2dfa2c91c0 --- /dev/null +++ b/src/it/benchmark/src/test/java/benchmark/DontRunMeTest.java @@ -0,0 +1,10 @@ +package benchmark; + +import org.junit.Test; + +public class DontRunMeTest { + @Test + public void dontRunThis() throws Exception { + throw new Exception("Normal tests should not be run with benchmarks."); + } +} diff --git a/src/it/benchmark/src/test/java/benchmark/TestBenchmark.java b/src/it/benchmark/src/test/java/benchmark/TestBenchmark.java new file mode 100644 index 0000000000..61c50f1403 --- /dev/null +++ b/src/it/benchmark/src/test/java/benchmark/TestBenchmark.java @@ -0,0 +1,16 @@ +package benchmark; + +import org.junit.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class TestBenchmark { + @Test + public void runThis() throws Exception { + // create a temporary folder to show that this method has executed + Path path = Paths.get("target/benchmark-run"); + Files.createDirectories(path); + } +} From e58d05b35c0940b0c09c3900a14c05f3ccfc7f5b Mon Sep 17 00:00:00 2001 From: Abhyudaya Sharma Date: Wed, 5 Jun 2019 17:37:26 +0530 Subject: [PATCH 3/5] Add documentation about `benchmark` profile to README.md --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 8e9435a786..7cb52e2b94 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,18 @@ If you had a `jar:test-jar` execution, delete it and add to `properties`: You can configure your plugin to treat every commit as a release candidate. See [Incrementals](https://github.com/jenkinsci/incrementals-tools) for details. +## Running Benchmarks + +To run JMH benchmarks from JUnit tests, you must run you must activate the `benchmark` +profile. For example: +```bash +mvn -Dbenchmark test +``` +When the `benchmark` property is set, no tests apart from JMH benchmarks will be run. +The names of the classes containing the benchmark runners should either begin with or +end with the the word `Benchmark`. For example, `FooBenchmark` and `BenchmarkFoo` will +be detected when using `-Dbenchmark`, however, `FooBar` will be ignored. + ## Baselines It is handy to be able to select different Jenkins baselines with a Maven profile. From fad617ac70d1ab5b42105bb45b79a38731dfaae3 Mon Sep 17 00:00:00 2001 From: Abhyudaya Sharma Date: Tue, 11 Jun 2019 10:30:27 +0530 Subject: [PATCH 4/5] Run benchmarks with incremental produced by jenkins-test-harness#135 --- Jenkinsfile | 2 +- pom.xml | 2 +- src/it/benchmark/postbuild.groovy | 3 +- .../test/java/benchmark/BenchmarkRunner.java | 32 +++++++++++++++++++ .../test/java/benchmark/SampleBenchmark.java | 18 +++++++++++ .../test/java/benchmark/TestBenchmark.java | 16 ---------- 6 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 src/it/benchmark/src/test/java/benchmark/BenchmarkRunner.java create mode 100644 src/it/benchmark/src/test/java/benchmark/SampleBenchmark.java delete mode 100644 src/it/benchmark/src/test/java/benchmark/TestBenchmark.java diff --git a/Jenkinsfile b/Jenkinsfile index d205b5663d..757c3f6889 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ pipeline { stages { stage('main') { steps { - sh 'mvn -B -Prun-plugin-pom-its clean verify' + sh 'mvn -B -Prun-plugin-pom-its -Pconsume-incrementals clean verify' } post { failure { diff --git a/pom.xml b/pom.xml index 305a3ee0cf..75c2dbae2d 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ ${jenkins.version} ${jenkins.version} - 2.49 + 2.50-rc1168.d591189fccf7 3.5 1.17 you-must-override-the-java.level-property diff --git a/src/it/benchmark/postbuild.groovy b/src/it/benchmark/postbuild.groovy index dc7cf15884..9e3e2089d9 100644 --- a/src/it/benchmark/postbuild.groovy +++ b/src/it/benchmark/postbuild.groovy @@ -1,4 +1,3 @@ // check if the benchmark was run -def file = new File(basedir, 'target/benchmark-run') +def file = new File(basedir, 'jmh-report.json') assert file.exists() -file.deleteDir() diff --git a/src/it/benchmark/src/test/java/benchmark/BenchmarkRunner.java b/src/it/benchmark/src/test/java/benchmark/BenchmarkRunner.java new file mode 100644 index 0000000000..0b6aeda151 --- /dev/null +++ b/src/it/benchmark/src/test/java/benchmark/BenchmarkRunner.java @@ -0,0 +1,32 @@ +package benchmark; + +import jenkins.benchmark.jmh.BenchmarkFinder; +import org.junit.Test; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.concurrent.TimeUnit; + +public class BenchmarkRunner { + @Test + public void runThis() throws Exception { + // number of iterations is kept to a minimum just to verify that the benchmarks work without spending extra + // time during builds. + ChainedOptionsBuilder optionsBuilder = + new OptionsBuilder() + .forks(1) + .warmupIterations(1) + .warmupBatchSize(1) + .measurementIterations(1) + .measurementBatchSize(1) + .shouldFailOnError(true) + .result("jmh-report.json") + .timeUnit(TimeUnit.MICROSECONDS) + .resultFormat(ResultFormatType.JSON); + BenchmarkFinder finder = new BenchmarkFinder(this.getClass().getPackage().getName()); + finder.findBenchmarks(optionsBuilder); + new Runner(optionsBuilder.build()).run(); + } +} diff --git a/src/it/benchmark/src/test/java/benchmark/SampleBenchmark.java b/src/it/benchmark/src/test/java/benchmark/SampleBenchmark.java new file mode 100644 index 0000000000..49e5599fc1 --- /dev/null +++ b/src/it/benchmark/src/test/java/benchmark/SampleBenchmark.java @@ -0,0 +1,18 @@ +package benchmark; + +import jenkins.benchmark.jmh.JmhBenchmark; +import jenkins.benchmark.jmh.JmhBenchmarkState; +import org.openjdk.jmh.annotations.Benchmark; + +import java.io.IOException; + +@JmhBenchmark +public class SampleBenchmark { + public static class MyState extends JmhBenchmarkState { + } + + @Benchmark + public void benchmark(MyState state) throws IOException { + state.getJenkins().setSystemMessage("Hello world"); + } +} diff --git a/src/it/benchmark/src/test/java/benchmark/TestBenchmark.java b/src/it/benchmark/src/test/java/benchmark/TestBenchmark.java deleted file mode 100644 index 61c50f1403..0000000000 --- a/src/it/benchmark/src/test/java/benchmark/TestBenchmark.java +++ /dev/null @@ -1,16 +0,0 @@ -package benchmark; - -import org.junit.Test; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -public class TestBenchmark { - @Test - public void runThis() throws Exception { - // create a temporary folder to show that this method has executed - Path path = Paths.get("target/benchmark-run"); - Files.createDirectories(path); - } -} From 8b926e71fd1037a82c78f68b8d06c855617959da Mon Sep 17 00:00:00 2001 From: Abhyudaya Sharma Date: Fri, 14 Jun 2019 09:23:34 +0530 Subject: [PATCH 5/5] Use released version of Jenkins Test Harness --- Jenkinsfile | 2 +- README.md | 2 ++ pom.xml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 757c3f6889..d205b5663d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ pipeline { stages { stage('main') { steps { - sh 'mvn -B -Prun-plugin-pom-its -Pconsume-incrementals clean verify' + sh 'mvn -B -Prun-plugin-pom-its clean verify' } post { failure { diff --git a/README.md b/README.md index 7cb52e2b94..9349124bae 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ The names of the classes containing the benchmark runners should either begin wi end with the the word `Benchmark`. For example, `FooBenchmark` and `BenchmarkFoo` will be detected when using `-Dbenchmark`, however, `FooBar` will be ignored. +See also: [documentation for JMH benchmarks](https://github.com/jenkinsci/jenkins-test-harness/blob/master/docs/jmh-benchmarks.adoc) + ## Baselines It is handy to be able to select different Jenkins baselines with a Maven profile. diff --git a/pom.xml b/pom.xml index 75c2dbae2d..f5d74dbce5 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ ${jenkins.version} ${jenkins.version} - 2.50-rc1168.d591189fccf7 + 2.50 3.5 1.17 you-must-override-the-java.level-property