diff --git a/docs/jmh-benchmarks.adoc b/docs/jmh-benchmarks.adoc new file mode 100644 index 00000000..1242de89 --- /dev/null +++ b/docs/jmh-benchmarks.adoc @@ -0,0 +1,45 @@ += JMH benchmarks with Jenkins +:toc: + +link:https://openjdk.java.net/projects/code-tools/jmh/[Java Microbenchmark Harness] allows running benchmarks +in the JVM. To run a benchmark where you need a Jenkins instance, you can use use link:../src/main/java/jenkins/benchmark/jmh/JmhBenchmarkState.java[``JmhBenchmarkState``] +as a state in your benchmark. This creates a temporary Jenkins instance for each fork of the JMH benchmark. + +== Writing benchmarks + +A reference to the Jenkins instance is available through either the `JmhBenchmarkState#getJenkins()` or through +`Jenkins.getInstance()` like you would otherwise do. `JmhBenchmarkState` provides `setup()` and `tearDown` methods +which can be overridden to configure the Jenkins instance according to your benchmark's requirements. + +== Running the benchmarks + +The benchmarks can be run through JUnit tests. From a test method, you can use the `OptionsBuilder` provided by JMH to +configure your benchmarks. For a sample, take a look at link:../src/test/java/jenkins/benchmark/jmh/BenchmarkTest.java[this]. +Classes containing benchmarks are found automatically by the `BenchmarkFinder` when annotated +with `@JmhBenchmark`. Benchmark reports can also be generated and can be visualized using the jmh-report plugin. + +NOTE: Benchmark methods need to be annotated by `@Benchmark` for JMH to detect them. + +== Sample benchmarks + +=== Simplest Benchmark: + +[source,java] +---- +@JmhBenchmark +public class JmhStateBenchmark { + public static class MyState extends JmhBenchmarkState { + } + + @Benchmark + public void benchmark(MyState state) { + // benchmark code goes here + } +} +---- + +=== Examples + +Some benchmarks have been implemented in the https://github.com/jenkinsci/role-strategy-plugin/tree/master/src/test/java/jmh/benchmarks[Role Strategy Plugin] +which show setting up the benchmarks for many different situations. + diff --git a/pom.xml b/pom.xml index 0825d86a..fe1e0309 100644 --- a/pom.xml +++ b/pom.xml @@ -85,8 +85,7 @@ THE SOFTWARE. org.jenkins-ci.main jenkins-war - - 1.651.2 + 2.60.3 executable-war @@ -167,8 +166,22 @@ THE SOFTWARE. 1.0.2 test + + org.openjdk.jmh + jmh-core + 1.21 + + + org.openjdk.jmh + jmh-generator-annprocess + 1.21 + + + org.reflections + reflections + 0.9.11 + - diff --git a/src/main/java/jenkins/benchmark/jmh/BenchmarkFinder.java b/src/main/java/jenkins/benchmark/jmh/BenchmarkFinder.java new file mode 100644 index 00000000..2bebebdf --- /dev/null +++ b/src/main/java/jenkins/benchmark/jmh/BenchmarkFinder.java @@ -0,0 +1,40 @@ +package jenkins.benchmark.jmh; + +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.reflections.Reflections; + +import java.util.Objects; +import java.util.Set; + +/** + * Find classes annotated with {@link JmhBenchmark} to run their benchmark methods. + * @since TODO + */ +public final class BenchmarkFinder { + final private String[] packageName; + + /** + * Creates a {@link BenchmarkFinder} + * + * @param packageNames find benchmarks in these packages + */ + public BenchmarkFinder(String... packageNames) { + this.packageName = packageNames; + } + + /** + * Includes classes annotated with {@link JmhBenchmark} as candidates for JMH benchmarks. + * + * @param optionsBuilder the optionsBuilder used to build the benchmarks + */ + public void findBenchmarks(ChainedOptionsBuilder optionsBuilder) { + Reflections reflections = new Reflections((Object[]) packageName); + Set> benchmarkClasses = reflections.getTypesAnnotatedWith(JmhBenchmark.class); + benchmarkClasses.forEach(clazz -> { + JmhBenchmark annotation = clazz.getAnnotation(JmhBenchmark.class); + if (Objects.nonNull(annotation)) { + optionsBuilder.include(clazz.getName() + annotation.value()); + } + }); + } +} diff --git a/src/main/java/jenkins/benchmark/jmh/JmhBenchmark.java b/src/main/java/jenkins/benchmark/jmh/JmhBenchmark.java new file mode 100644 index 00000000..1da307ec --- /dev/null +++ b/src/main/java/jenkins/benchmark/jmh/JmhBenchmark.java @@ -0,0 +1,25 @@ +package jenkins.benchmark.jmh; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotate your benchmark classes with this annotation to allow them to be discovered by {@link BenchmarkFinder} + * @since TODO + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface JmhBenchmark { + /** + * Methods which annotated by {@link org.openjdk.jmh.annotations.Benchmark} + * in classes annotated by {@link JmhBenchmark} are to be run as benchmarks if they + * match this regex pattern. + *

+ * Matches all functions by default, i.e. default pattern is {@code .*}. + * + * @return the regular expression used to match function names. + */ + String value() default ".*"; +} diff --git a/src/main/java/jenkins/benchmark/jmh/JmhBenchmarkState.java b/src/main/java/jenkins/benchmark/jmh/JmhBenchmarkState.java new file mode 100644 index 00000000..7b262f63 --- /dev/null +++ b/src/main/java/jenkins/benchmark/jmh/JmhBenchmarkState.java @@ -0,0 +1,158 @@ +package jenkins.benchmark.jmh; + +import hudson.model.Hudson; +import hudson.model.RootAction; +import hudson.security.ACL; +import jenkins.model.Jenkins; +import jenkins.model.JenkinsLocationConfiguration; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.eclipse.jetty.server.Server; +import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.TemporaryDirectoryAllocator; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; + +import javax.annotation.CheckForNull; +import javax.servlet.ServletContext; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Standard benchmark {@link State} for JMH when a Jenkins instance is required. + *

+ * To use a Jenkins instance in your benchmark, your class containing benchmarks should have a public static inner + * class that extends this class and should be annotated with {@link JmhBenchmark} to allow it to be automatically + * discovered by {@link BenchmarkFinder}. To configure the instance, use {@link #setup()}. + * + * @see #setup() + * @see #tearDown() + * @see BenchmarkFinder + * @since TODO + */ +@State(Scope.Benchmark) +public abstract class JmhBenchmarkState implements RootAction { + private static final Logger LOGGER = Logger.getLogger(JmhBenchmarkState.class.getName()); + private static final String contextPath = "/jenkins"; + + private final TemporaryDirectoryAllocator temporaryDirectoryAllocator = new TemporaryDirectoryAllocator(); + private final MutableInt localPort = new MutableInt(); + + private Jenkins jenkins = null; + private Server server = null; + + /** + * Sets up the temporary Jenkins instance for benchmarks. + *

+ * One Jenkins instance is created for each fork of the benchmark. + * + * @throws Exception if unable to start the instance. + */ + @Setup(org.openjdk.jmh.annotations.Level.Trial) + public final void setupJenkins() throws Exception { + // Set the jenkins.install.InstallState TEST to emulate + // org.jvnet.hudson.test.JenkinsRule behaviour and avoid manual + // security setup as in a default installation. + System.setProperty("jenkins.install.state", "TEST"); + launchInstance(); + ACL.impersonate(ACL.SYSTEM); + setup(); + } + + /** + * Terminates the jenkins instance after the benchmark has completed its execution. + * Run once for each Jenkins that was started. + */ + @TearDown(org.openjdk.jmh.annotations.Level.Trial) + public final void terminateJenkins() { + try { + tearDown(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Exception occurred during tearDown of Jenkins instance", e); + } finally { + JenkinsRule._stopJenkins(server, null, jenkins); + try { + temporaryDirectoryAllocator.dispose(); + } catch (InterruptedException | IOException e) { + LOGGER.log(Level.WARNING, "Unable to dispose temporary Jenkins directory" + + "that was started for benchmark", e); + } + } + } + + private void launchInstance() throws Exception { + ImmutablePair results = JenkinsRule._createWebServer(contextPath, localPort::setValue, + getClass().getClassLoader(), localPort.getValue(), JenkinsRule::_configureUserRealm); + + server = results.left; + ServletContext webServer = results.right; + + jenkins = new Hudson(temporaryDirectoryAllocator.allocate(), webServer); + JenkinsRule._configureJenkinsForTest(jenkins); + JenkinsRule._configureUpdateCenter(jenkins); + jenkins.getActions().add(this); + + String url = Objects.requireNonNull(getJenkinsURL()).toString(); + Objects.requireNonNull(JenkinsLocationConfiguration.get()).setUrl(url); + LOGGER.log(Level.INFO, "Running on {0}", url); + } + + private URL getJenkinsURL() throws MalformedURLException { + return new URL("http://localhost:" + localPort.getValue() + contextPath + "/"); + } + + /** + * Get reference to the {@link Jenkins} started for the benchmark. + *

+ * The instance can also be obtained using {@link Jenkins#getInstanceOrNull()} + * + * @return the Jenkins instance started for the benchmark. + */ + public Jenkins getJenkins() { + return jenkins; + } + + /** + * Override to setup resources required for the benchmark. + *

+ * Runs before the benchmarks are run. At this state, the Jenkins instance + * is ready to be worked upon and is available using {@link #getJenkins()}. + * Does nothing by default. + */ + public void setup() throws Exception { + // noop + } + + /** + * Override to perform cleanup of resource initialized during setup. + *

+ * Run before the Jenkins instance is terminated. Does nothing by default. + */ + public void tearDown() { + // noop + } + + @CheckForNull + @Override + public String getIconFileName() { + return null; + } + + @CheckForNull + @Override + public String getDisplayName() { + return null; + } + + @CheckForNull + @Override + public String getUrlName() { + return "self"; + } +} diff --git a/src/main/java/org/jvnet/hudson/test/JenkinsRule.java b/src/main/java/org/jvnet/hudson/test/JenkinsRule.java index 739c838b..7ac7aa94 100644 --- a/src/main/java/org/jvnet/hudson/test/JenkinsRule.java +++ b/src/main/java/org/jvnet/hudson/test/JenkinsRule.java @@ -159,6 +159,8 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.jar.Manifest; import java.util.logging.Filter; import java.util.logging.Level; @@ -185,6 +187,7 @@ import org.apache.commons.beanutils.PropertyUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.eclipse.jetty.http.MimeTypes; import org.eclipse.jetty.security.HashLoginService; import org.eclipse.jetty.security.LoginService; @@ -410,17 +413,9 @@ public void before() throws Throwable { f.set(null,null); throw e; } - jenkins.setNoUsageStatistics(true); // collecting usage stats from tests are pointless. - - jenkins.setCrumbIssuer(new TestCrumbIssuer()); - - jenkins.servletContext.setAttribute("app",jenkins); - jenkins.servletContext.setAttribute("version","?"); - WebAppMain.installExpressionFactory(new ServletContextEvent(jenkins.servletContext)); - - // set a default JDK to be the one that the harness is using. - jenkins.getJDKs().add(new JDK("default",System.getProperty("java.home"))); + jenkins.setCrumbIssuer(new TestCrumbIssuer()); // TODO: Move to _configureJenkinsForTest after JENKINS-55240 + _configureJenkinsForTest(jenkins); configureUpdateCenter(); // expose the test instance as a part of URL tree. @@ -430,11 +425,45 @@ public void before() throws Throwable { JenkinsLocationConfiguration.get().setUrl(getURL().toString()); } + /** + * Configures a Jenkins instance for test. + * + * @param jenkins jenkins instance which has to be configured + * @throws Exception if unable to configure + * @since TODO + */ + public static void _configureJenkinsForTest(Jenkins jenkins) throws Exception { + jenkins.setNoUsageStatistics(true); // collecting usage stats from tests is pointless. + jenkins.servletContext.setAttribute("app", jenkins); + jenkins.servletContext.setAttribute("version", "?"); + WebAppMain.installExpressionFactory(new ServletContextEvent(jenkins.servletContext)); + + // set a default JDK to be the one that the harness is using. + jenkins.getJDKs().add(new JDK("default", System.getProperty("java.home"))); + } + + private static void dumpThreads() { + ThreadInfo[] threadInfos = Functions.getThreadInfos(); + Functions.ThreadGroupMap m = Functions.sortThreadsAndGetGroupMap(threadInfos); + for (ThreadInfo ti : threadInfos) { + System.err.println(Functions.dumpThreadInfo(ti, m)); + } + } + /** * Configures the update center setting for the test. * By default, we load updates from local proxy to avoid network traffic as much as possible. */ protected void configureUpdateCenter() throws Exception { + _configureUpdateCenter(jenkins); + } + + /** + * Internal method used to configure update center to avoid network traffic. + * @param jenkins the Jenkins to configure + * @since TODO + */ + public static void _configureUpdateCenter(Jenkins jenkins) throws Exception { final String updateCenterUrl; jettyLevel(Level.WARNING); try { @@ -451,14 +480,6 @@ protected void configureUpdateCenter() throws Exception { sites.clear(); sites.add(new UpdateSite("default", updateCenterUrl)); } - - private static void dumpThreads() { - ThreadInfo[] threadInfos = Functions.getThreadInfos(); - Functions.ThreadGroupMap m = Functions.sortThreadsAndGetGroupMap(threadInfos); - for (ThreadInfo ti : threadInfos) { - System.err.println(Functions.dumpThreadInfo(ti, m)); - } - } /** * Override to tear down your specific external resource. @@ -483,25 +504,7 @@ public void after() throws Exception { clients.clear(); } finally { - jettyLevel(Level.WARNING); - try { - server.stop(); - } catch (Exception e) { - // ignore - } finally { - jettyLevel(Level.INFO); - } - for (LenientRunnable r : tearDowns) - try { - r.run(); - } catch (Exception e) { - // ignore - } - - if (jenkins!=null) - jenkins.cleanUp(); - ExtensionList.clearLegacyInstances(); - DescriptorExtensionList.clearLegacyInstances(); + _stopJenkins(server, tearDowns, jenkins); try { env.dispose(); @@ -524,6 +527,46 @@ public void after() throws Exception { } } + /** + * Internal method to stop Jenkins instance. + * + * @param server server on which Jenkins is running. + * @param tearDowns tear down methods for tests + * @param jenkins the jenkins instance + * @since TODO + */ + public static void _stopJenkins(Server server, List tearDowns, Jenkins jenkins) { + final RuntimeException exception = new RuntimeException("One or more problems while shutting down Jenkins"); + + jettyLevel(Level.WARNING); + try { + server.stop(); + } catch (Exception e) { + exception.addSuppressed(e); + } finally { + jettyLevel(Level.INFO); + } + + if (tearDowns != null) { + for (LenientRunnable r : tearDowns) { + try { + r.run(); + } catch (Exception e) { + exception.addSuppressed(e); + } + } + } + + if (jenkins != null) + jenkins.cleanUp(); + ExtensionList.clearLegacyInstances(); + DescriptorExtensionList.clearLegacyInstances(); + + if (exception.getSuppressed().length > 0) { + throw exception; + } + } + private static void jettyLevel(Level level) { Logger.getLogger("org.eclipse.jetty").setLevel(level); } @@ -671,7 +714,29 @@ public File getWebAppRoot() throws Exception { * that we need for testing. */ protected ServletContext createWebServer() throws Exception { - server = new Server(new ThreadPoolImpl(new ThreadPoolExecutor(10, 10, 10L, TimeUnit.SECONDS, new LinkedBlockingQueue(),new ThreadFactory() { + ImmutablePair results = _createWebServer(contextPath, + (x) -> localPort = x, getClass().getClassLoader(), localPort, this::configureUserRealm); + server = results.left; + LOGGER.log(Level.INFO, "Running on {0}", getURL()); + return results.right; + } + + /** + * Creates a web server on which Jenkins can run + * + * @param contextPath the context path at which to put Jenkins + * @param portSetter the port on which the server runs will be set using this function + * @param classLoader the class loader for the {@link WebAppContext} + * @param localPort port on which the server runs + * @param loginServiceSupplier configures the {@link LoginService} for the instance + * @return ImmutablePair consisting of the {@link Server} and the {@link ServletContext} + * @since TODO + */ + public static ImmutablePair _createWebServer(String contextPath, Consumer portSetter, + ClassLoader classLoader, int localPort, + Supplier loginServiceSupplier) + throws Exception { + Server server = new Server(new ThreadPoolImpl(new ThreadPoolExecutor(10, 10, 10L, TimeUnit.SECONDS, new LinkedBlockingQueue(), new ThreadFactory() { public Thread newThread(Runnable r) { Thread t = new Thread(r); t.setName("Jetty Thread Pool"); @@ -680,12 +745,12 @@ public Thread newThread(Runnable r) { }))); WebAppContext context = new WebAppContext(WarExploder.getExplodedDir().getPath(), contextPath); - context.setClassLoader(getClass().getClassLoader()); + context.setClassLoader(classLoader); context.setConfigurations(new Configuration[]{new WebXmlConfiguration()}); context.addBean(new NoListenerConfiguration(context)); server.setHandler(context); context.setMimeTypes(MIME_TYPES); - context.getSecurityHandler().setLoginService(configureUserRealm()); + context.getSecurityHandler().setLoginService(loginServiceSupplier.get()); context.setResourceBase(WarExploder.getExplodedDir().getPath()); ServerConnector connector = new ServerConnector(server, 1, 1); @@ -702,16 +767,27 @@ public Thread newThread(Runnable r) { server.addConnector(connector); server.start(); - localPort = connector.getLocalPort(); - LOGGER.log(Level.INFO, "Running on {0}", getURL()); + portSetter.accept(connector.getLocalPort()); - return context.getServletContext(); + ServletContext servletContext = context.getServletContext(); + return new ImmutablePair<>(server, servletContext); } /** * Configures a security realm for a test. */ protected LoginService configureUserRealm() { + return _configureUserRealm(); + } + + /** + * Creates a {@link HashLoginService} with three users: alice, bob and charlie + * + * The password is same as the username + * @return a new login service + * @since TODO + */ + public static LoginService _configureUserRealm() { HashLoginService realm = new HashLoginService(); realm.setName("default"); // this is the magic realm name to make it effective on everywhere UserStore userStore = new UserStore(); @@ -2126,7 +2202,7 @@ public WebResponse loadWebResponse(final WebRequest webRequest) throws IOExcepti public WebClient login(String username, String password) throws Exception { return login(username,password,false); } - + /** * Returns {@code true} if JavaScript is enabled and the script engine was loaded successfully. * Short-hand method to ease discovery of feature + improve readability diff --git a/src/test/java/jenkins/benchmark/jmh/BenchmarkTest.java b/src/test/java/jenkins/benchmark/jmh/BenchmarkTest.java new file mode 100644 index 00000000..532a4efb --- /dev/null +++ b/src/test/java/jenkins/benchmark/jmh/BenchmarkTest.java @@ -0,0 +1,41 @@ +package jenkins.benchmark.jmh; + +import org.junit.Test; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +/** + * Runs sample benchmarks from JUnit tests. + */ +public class BenchmarkTest { + @Test + public void testJmhBenchmarks() throws Exception { + // create directory for JMH reports + Path path = Paths.get("target/jmh-reports/"); + Files.createDirectories(path); + + // number of iterations is kept to a minimum just to verify that the benchmarks work without spending extra + // time during builds. + ChainedOptionsBuilder optionsBuilder = + new OptionsBuilder() + .forks(1) + .warmupIterations(1) + .warmupBatchSize(1) + .measurementIterations(1) + .measurementBatchSize(1) + .shouldFailOnError(true) + .result("target/jmh-reports/jmh-benchmark-report.json") + .timeUnit(TimeUnit.MICROSECONDS) + .resultFormat(ResultFormatType.JSON); + BenchmarkFinder finder = new BenchmarkFinder(this.getClass().getPackage().getName()); + finder.findBenchmarks(optionsBuilder); + new Runner(optionsBuilder.build()).run(); + } +} diff --git a/src/test/java/jenkins/benchmark/jmh/samples/JmhStateBenchmark.java b/src/test/java/jenkins/benchmark/jmh/samples/JmhStateBenchmark.java new file mode 100644 index 00000000..6ec00da3 --- /dev/null +++ b/src/test/java/jenkins/benchmark/jmh/samples/JmhStateBenchmark.java @@ -0,0 +1,21 @@ +package jenkins.benchmark.jmh.samples; + +import jenkins.benchmark.jmh.JmhBenchmark; +import jenkins.benchmark.jmh.JmhBenchmarkState; +import org.openjdk.jmh.annotations.Benchmark; + +import java.util.Objects; + +/** + * Sample benchmark without doing anything special to the Jenkins instance. + */ +@JmhBenchmark +public class JmhStateBenchmark { + public static class MyState extends JmhBenchmarkState { + } + + @Benchmark + public void benchmark(MyState state) { + Objects.requireNonNull(state.getJenkins()); + } +}