Merge pull request #135 from AbhyudayaSharma/jmh

[JENKINS-57653] Introduce JMH benchmarks to Jenkins Test Harness

docs/jmh-benchmarks.adoc

@@ -0,0 +1,45 @@
+= JMH benchmarks with Jenkins
+:toc:
+
+link:https://openjdk.java.net/projects/code-tools/jmh/[Java Microbenchmark Harness] allows running benchmarks
+in the JVM. To run a benchmark where you need a Jenkins instance, you can use use link:../src/main/java/jenkins/benchmark/jmh/JmhBenchmarkState.java[``JmhBenchmarkState``]
+as a state in your benchmark. This creates a temporary Jenkins instance for each fork of the JMH benchmark.
+
+== Writing benchmarks
+
+A reference to the Jenkins instance is available through either the `JmhBenchmarkState#getJenkins()` or through
+`Jenkins.getInstance()` like you would otherwise do. `JmhBenchmarkState` provides `setup()` and `tearDown` methods
+which can be overridden to configure the Jenkins instance according to your benchmark's requirements.
+
+== Running the benchmarks
+
+The benchmarks can be run through JUnit tests. From a test method, you can use the `OptionsBuilder` provided by JMH to
+configure your benchmarks. For a sample, take a look at link:../src/test/java/jenkins/benchmark/jmh/BenchmarkTest.java[this].
+Classes containing benchmarks are found automatically by the `BenchmarkFinder` when annotated
+with `@JmhBenchmark`. Benchmark reports can also be generated and can be visualized using the jmh-report plugin.
+
+NOTE: Benchmark methods need to be annotated by `@Benchmark` for JMH to detect them.
+
+== Sample benchmarks
+
+=== Simplest Benchmark:
+
+[source,java]
+----
+@JmhBenchmark
+public class JmhStateBenchmark {
+    public static class MyState extends JmhBenchmarkState {
+    }
+
+    @Benchmark
+    public void benchmark(MyState state) {
+        // benchmark code goes here
+    }
+}
+----
+
+=== Examples
+
+Some benchmarks have been implemented in the https://github.com/jenkinsci/role-strategy-plugin/tree/master/src/test/java/jmh/benchmarks[Role Strategy Plugin]
+which show setting up the benchmarks for many different situations.
+

pom.xml

@@ -85,8 +85,7 @@ THE SOFTWARE.
     <dependency>
       <groupId>org.jenkins-ci.main</groupId>
       <artifactId>jenkins-war</artifactId>
-      <!--to have access to User.getById-->
-      <version>1.651.2</version>
+      <version>2.60.3</version>
       <type>executable-war</type>
       <exclusions>
         <exclusion>
@@ -167,8 +166,22 @@ THE SOFTWARE.
       <version>1.0.2</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-core</artifactId>
+      <version>1.21</version>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+      <version>1.21</version>
+    </dependency>
+    <dependency>
+      <groupId>org.reflections</groupId>
+      <artifactId>reflections</artifactId>
+      <version>0.9.11</version>
+    </dependency>
   </dependencies>
-
   <build>
     <plugins>
       <plugin>

src/main/java/jenkins/benchmark/jmh/BenchmarkFinder.java

@@ -0,0 +1,40 @@
+package jenkins.benchmark.jmh;
+
+import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
+import org.reflections.Reflections;
+
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Find classes annotated with {@link JmhBenchmark} to run their benchmark methods.
+ * @since TODO
+ */
+public final class BenchmarkFinder {
+    final private String[] packageName;
+
+    /**
+     * Creates a {@link BenchmarkFinder}
+     *
+     * @param packageNames find benchmarks in these packages
+     */
+    public BenchmarkFinder(String... packageNames) {
+        this.packageName = packageNames;
+    }
+
+    /**
+     * Includes classes annotated with {@link JmhBenchmark} as candidates for JMH benchmarks.
+     *
+     * @param optionsBuilder the optionsBuilder used to build the benchmarks
+     */
+    public void findBenchmarks(ChainedOptionsBuilder optionsBuilder) {
+        Reflections reflections = new Reflections((Object[]) packageName);
+        Set<Class<?>> benchmarkClasses = reflections.getTypesAnnotatedWith(JmhBenchmark.class);
+        benchmarkClasses.forEach(clazz -> {
+            JmhBenchmark annotation = clazz.getAnnotation(JmhBenchmark.class);
+            if (Objects.nonNull(annotation)) {
+                optionsBuilder.include(clazz.getName() + annotation.value());
+            }
+        });
+    }
+}

src/main/java/jenkins/benchmark/jmh/JmhBenchmark.java

@@ -0,0 +1,25 @@
+package jenkins.benchmark.jmh;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotate your benchmark classes with this annotation to allow them to be discovered by {@link BenchmarkFinder}
+ * @since TODO
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface JmhBenchmark {
+    /**
+     * Methods which annotated by {@link org.openjdk.jmh.annotations.Benchmark}
+     * in classes annotated by {@link JmhBenchmark} are to be run as benchmarks if they
+     * match this regex pattern.
+     * <p>
+     * Matches all functions by default, i.e. default pattern is {@code .*}.
+     *
+     * @return the regular expression used to match function names.
+     */
+    String value() default ".*";
+}

src/main/java/jenkins/benchmark/jmh/JmhBenchmarkState.java

@@ -0,0 +1,158 @@
+package jenkins.benchmark.jmh;
+
+import hudson.model.Hudson;
+import hudson.model.RootAction;
+import hudson.security.ACL;
+import jenkins.model.Jenkins;
+import jenkins.model.JenkinsLocationConfiguration;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.eclipse.jetty.server.Server;
+import org.jvnet.hudson.test.JenkinsRule;
+import org.jvnet.hudson.test.TemporaryDirectoryAllocator;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+import javax.annotation.CheckForNull;
+import javax.servlet.ServletContext;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Objects;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Standard benchmark {@link State} for JMH when a Jenkins instance is required.
+ * <p>
+ * To use a Jenkins instance in your benchmark, your class containing benchmarks should have a public static inner
+ * class that extends this class and should be annotated with {@link JmhBenchmark} to allow it to be automatically
+ * discovered by {@link BenchmarkFinder}. To configure the instance, use {@link #setup()}.
+ *
+ * @see #setup()
+ * @see #tearDown()
+ * @see BenchmarkFinder
+ * @since TODO
+ */
+@State(Scope.Benchmark)
+public abstract class JmhBenchmarkState implements RootAction {
+    private static final Logger LOGGER = Logger.getLogger(JmhBenchmarkState.class.getName());
+    private static final String contextPath = "/jenkins";
+
+    private final TemporaryDirectoryAllocator temporaryDirectoryAllocator = new TemporaryDirectoryAllocator();
+    private final MutableInt localPort = new MutableInt();
+
+    private Jenkins jenkins = null;
+    private Server server = null;
+
+    /**
+     * Sets up the temporary Jenkins instance for benchmarks.
+     * <p>
+     * One Jenkins instance is created for each fork of the benchmark.
+     *
+     * @throws Exception if unable to start the instance.
+     */
+    @Setup(org.openjdk.jmh.annotations.Level.Trial)
+    public final void setupJenkins() throws Exception {
+        // Set the jenkins.install.InstallState TEST to emulate
+        // org.jvnet.hudson.test.JenkinsRule behaviour and avoid manual
+        // security setup as in a default installation.
+        System.setProperty("jenkins.install.state", "TEST");
+        launchInstance();
+        ACL.impersonate(ACL.SYSTEM);
+        setup();
+    }
+
+    /**
+     * Terminates the jenkins instance after the benchmark has completed its execution.
+     * Run once for each Jenkins that was started.
+     */
+    @TearDown(org.openjdk.jmh.annotations.Level.Trial)
+    public final void terminateJenkins() {
+        try {
+            tearDown();
+        } catch (Exception e) {
+            LOGGER.log(Level.SEVERE, "Exception occurred during tearDown of Jenkins instance", e);
+        } finally {
+            JenkinsRule._stopJenkins(server, null, jenkins);
+            try {
+                temporaryDirectoryAllocator.dispose();
+            } catch (InterruptedException | IOException e) {
+                LOGGER.log(Level.WARNING, "Unable to dispose temporary Jenkins directory" +
+                                                  "that was started for benchmark", e);
+            }
+        }
+    }
+
+    private void launchInstance() throws Exception {
+        ImmutablePair<Server, ServletContext> results = JenkinsRule._createWebServer(contextPath, localPort::setValue,
+                getClass().getClassLoader(), localPort.getValue(), JenkinsRule::_configureUserRealm);
+
+        server = results.left;
+        ServletContext webServer = results.right;
+
+        jenkins = new Hudson(temporaryDirectoryAllocator.allocate(), webServer);
+        JenkinsRule._configureJenkinsForTest(jenkins);
+        JenkinsRule._configureUpdateCenter(jenkins);
+        jenkins.getActions().add(this);
+
+        String url = Objects.requireNonNull(getJenkinsURL()).toString();
+        Objects.requireNonNull(JenkinsLocationConfiguration.get()).setUrl(url);
+        LOGGER.log(Level.INFO, "Running on {0}", url);
+    }
+
+    private URL getJenkinsURL() throws MalformedURLException {
+        return new URL("http://localhost:" + localPort.getValue() + contextPath + "/");
+    }
+
+    /**
+     * Get reference to the {@link Jenkins} started for the benchmark.
+     * <p>
+     * The instance can also be obtained using {@link Jenkins#getInstanceOrNull()}
+     *
+     * @return the Jenkins instance started for the benchmark.
+     */
+    public Jenkins getJenkins() {
+        return jenkins;
+    }
+
+    /**
+     * Override to setup resources required for the benchmark.
+     * <p>
+     * Runs before the benchmarks are run. At this state, the Jenkins instance
+     * is ready to be worked upon and is available using {@link #getJenkins()}.
+     * Does nothing by default.
+     */
+    public void setup() throws Exception {
+        // noop
+    }
+
+    /**
+     * Override to perform cleanup of resource initialized during setup.
+     * <p>
+     * Run before the Jenkins instance is terminated. Does nothing by default.
+     */
+    public void tearDown() {
+        // noop
+    }
+
+    @CheckForNull
+    @Override
+    public String getIconFileName() {
+        return null;
+    }
+
+    @CheckForNull
+    @Override
+    public String getDisplayName() {
+        return null;
+    }
+
+    @CheckForNull
+    @Override
+    public String getUrlName() {
+        return "self";
+    }
+}