Benchmarking code to check for performance regressions

pom.xml

@@ -62,6 +62,7 @@
 		<module>quickfixj-examples</module>
 		<module>quickfixj-all</module>
 		<module>quickfixj-distribution</module>
+		<module>quickfixj-perf-test</module>
 	</modules>
 
 	<properties>
@@ -109,6 +110,7 @@
 		<org.quickfixj.orchestra.tools.version>1.0.2</org.quickfixj.orchestra.tools.version>
 		<proxool.version>0.9.1</proxool.version>
 		<jaxen.version>2.0.0</jaxen.version>
+        <jmh.version>1.36</jmh.version>
 	</properties>
 
    <dependencyManagement>

quickfixj-distribution/pom.xml

@@ -122,11 +122,11 @@
 			<artifactId>quickfixj-messages-fix40</artifactId>
 			<version>${project.version}</version>
 		</dependency>
-		<dependency>
-			<groupId>com.cloudhopper.proxool</groupId>
-			<artifactId>proxool</artifactId>
-			<optional>true</optional>
-		</dependency>
+    <dependency>
+			<groupId>org.quickfixj</groupId>
+			<artifactId>quickfixj-perf-test</artifactId>
+			<version>${project.version}</version>
+    </dependency>
 		<dependency>
 			<groupId>com.sleepycat</groupId>
 			<artifactId>je</artifactId>

quickfixj-perf-test/README.md

@@ -0,0 +1,55 @@
+# QuickFIX/J Performance test
+
+This is a [JMH](https://github.com/openjdk/jmh) benchmark module for QuickFIX/J FIX protocol implementation. 
+
+## How to run
+
+### Using your favorite IDE
+
+Performance regression classes can be individually run using your favorite IDE.
+
+### Creating executable jar for the performance testing
+
+Build executable jar using following maven command
+
+```
+$ mvn clean package    
+```
+
+Use following command to run complete set of performance regression test cases
+
+```
+$ java -jar target/quickfixj-perf-test.jar
+```
+
+You can list available performance benchmarks using `-l` option
+```
+$ java -jar target/quickfixj-perf-test.jar -l
+```
+
+You can run individual benchmarks by providing the class name or benchmark method name
+```
+$ java -jar target/quickfixj-perf-test.jar MessageCrackerPerfTest.crack
+
+$ java -jar target/quickfixj-perf-test.jar MessageCrackerPerfTest
+```
+
+You can change the time unit used in the benchmark test using `-tu` option
+Following command is an example of using micro second for describing benchmark test results.
+```
+$ java -jar target/quickfixj-perf-test.jar MessageCrackerPerfTest -tu us
+```
+
+For more available options use `-h` option
+
+```
+$ java -jar target/quickfixj-perf-test.jar -h
+```
+
+#### Guideline for future performance enhancements
+
+1. Check if there is already benchmark available for the code you are planning to optimize.
+2. If there is no benchmark code, first create a benchmark (Example `a-missing-perf-regression` branch) and make pull request to `master` branch.
+3. Make your performance improvement in a new branch (Example `a-perf-improvement` branch).
+4. When you make pull request provide comparison between current benchmark (`master`) and new benchmark values (`a-perf-improvement`)
+5. This means providing output of the benchmark execution in two branches in a PR comment

quickfixj-perf-test/pom.xml

@@ -0,0 +1,71 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.quickfixj</groupId>
+        <artifactId>quickfixj-parent</artifactId>
+        <version>3.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>quickfixj-perf-test</artifactId>
+    <packaging>jar</packaging>
+
+    <name>QuickFIX/J Performance test</name>
+    <description>QuickFIX/J Performance test for regression and improvements</description>
+    <url>http://www.quickfixj.org</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.quickfixj</groupId>
+            <artifactId>quickfixj-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.quickfixj</groupId>
+            <artifactId>quickfixj-messages-all</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>${jmh.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>${jmh.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>create-self-executing-jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <finalName>${project.artifactId}</finalName>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.openjdk.jmh.Main</mainClass>
+                                </transformer>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
+                            <shadedArtifactAttached>true</shadedArtifactAttached>
+                            <shadedClassifierName>standalone</shadedClassifierName>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

quickfixj-perf-test/src/main/java/org/quickfixj/AbstractPerfTest.java

@@ -0,0 +1,21 @@
+package org.quickfixj;
+
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@State(Scope.Benchmark)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@BenchmarkMode(value = {Mode.Throughput, Mode.SampleTime})
+abstract class AbstractPerfTest {
+}

quickfixj-perf-test/src/main/java/org/quickfixj/MessageCrackerPerfTest.java

@@ -0,0 +1,65 @@
+package org.quickfixj;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.quickfixj.sample.SampleQuickFixJApplication;
+import quickfix.ConfigError;
+import quickfix.DataDictionary;
+import quickfix.ValidationSettings;
+import quickfix.FieldNotFound;
+import quickfix.IncorrectTagValue;
+import quickfix.InvalidMessage;
+import quickfix.SessionID;
+import quickfix.UnsupportedMessageType;
+import quickfix.fix44.ExecutionReport;
+
+public class MessageCrackerPerfTest extends AbstractPerfTest {
+
+    private SessionID sessionID;
+    private ExecutionReport executionReport;
+
+    private SampleQuickFixJApplication application;
+
+    @Setup
+    public void prepare() throws ConfigError, InvalidMessage {
+
+        String data = "8=FIX.4.4\0019=309\00135=8\00149=ASX\00156=CL1_FIX44\00134=4\001" +
+                "52=20060324-01:05:58\00117=X-B-WOW-1494E9A0:58BD3F9D-1109\001150=D\001" +
+                "39=0\00111=184271\00138=200\001198=1494E9A0:58BD3F9D\001526=4324\001" +
+                "37=B-WOW-1494E9A0:58BD3F9D\00155=WOW\00154=1\001151=200\00114=0\00140=2\001" +
+                "44=15\00159=1\0016=0\001453=3\001448=AAA35791\001447=D\001452=3\001448=8\001" +
+                "447=D\001452=4\001448=FIX11\001447=D\001452=36\00160=20060320-03:34:29\00110=169\001";
+
+        sessionID = new SessionID("FIX4.4", "CL1_FIX44", "ASX");
+        executionReport = new ExecutionReport();
+        DataDictionary dataDictionary = new DataDictionary(MessageCrackerPerfTest.class.getClassLoader()
+                .getResourceAsStream("FIX44.xml"));
+        ValidationSettings validationSettings = new ValidationSettings();
+
+        executionReport.fromString(data, dataDictionary, validationSettings, false);
+
+        application = new SampleQuickFixJApplication();
+    }
+
+    @Benchmark
+    public void baseline() {
+    }
+
+    @Benchmark
+    public void crack() throws UnsupportedMessageType, IncorrectTagValue, FieldNotFound {
+        application.fromApp(executionReport, sessionID);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(".*" + MessageCrackerPerfTest.class.getSimpleName() + ".*")
+                .build();
+
+        new Runner(opt).run();
+    }
+
+}

quickfixj-perf-test/src/main/java/org/quickfixj/MessageParsePerfTest.java

@@ -0,0 +1,55 @@
+package org.quickfixj;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import quickfix.ConfigError;
+import quickfix.DataDictionary;
+import quickfix.ValidationSettings;
+import quickfix.InvalidMessage;
+import quickfix.fix44.ExecutionReport;
+
+public class MessageParsePerfTest extends AbstractPerfTest {
+
+    private ExecutionReport executionReport;
+    private DataDictionary dataDictionary;
+    private ValidationSettings validationSettings;
+    private String data;
+
+    @Setup
+    public void prepare() throws ConfigError {
+
+        data = "8=FIX.4.4\0019=309\00135=8\00149=ASX\00156=CL1_FIX44\00134=4\001" +
+                "52=20060324-01:05:58\00117=X-B-WOW-1494E9A0:58BD3F9D-1109\001150=D\001" +
+                "39=0\00111=184271\00138=200\001198=1494E9A0:58BD3F9D\001526=4324\001" +
+                "37=B-WOW-1494E9A0:58BD3F9D\00155=WOW\00154=1\001151=200\00114=0\00140=2\001" +
+                "44=15\00159=1\0016=0\001453=3\001448=AAA35791\001447=D\001452=3\001448=8\001" +
+                "447=D\001452=4\001448=FIX11\001447=D\001452=36\00160=20060320-03:34:29\00110=169\001";
+
+        executionReport = new ExecutionReport();
+        dataDictionary = new DataDictionary(MessageCrackerPerfTest.class.getClassLoader()
+                .getResourceAsStream("FIX44.xml"));
+        validationSettings = new ValidationSettings();
+    }
+
+    @Benchmark
+    public void baseline() {
+    }
+
+    @Benchmark
+    public void parse() throws InvalidMessage {
+        executionReport.fromString(data, dataDictionary, validationSettings, false);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(".*" + MessageParsePerfTest.class.getSimpleName() + ".*")
+                .build();
+
+        new Runner(opt).run();
+    }
+
+}

quickfixj-perf-test/src/main/java/org/quickfixj/sample/SampleQuickFixJApplication.java

@@ -0,0 +1,53 @@
+package org.quickfixj.sample;
+
+import quickfix.Application;
+import quickfix.FieldNotFound;
+import quickfix.IncorrectTagValue;
+import quickfix.Message;
+import quickfix.SessionID;
+import quickfix.UnsupportedMessageType;
+import quickfix.fix44.ExecutionReport;
+import quickfix.fix44.MessageCracker;
+
+public class SampleQuickFixJApplication extends MessageCracker implements Application {
+
+    @Override
+    public void onCreate(SessionID sessionId) {
+
+    }
+
+    @Override
+    public void onLogon(SessionID sessionId) {
+
+    }
+
+    @Override
+    public void onLogout(SessionID sessionId) {
+
+    }
+
+    @Override
+    public void toAdmin(Message message, SessionID sessionId) {
+
+    }
+
+    @Override
+    public void fromAdmin(Message message, SessionID sessionId) {
+
+    }
+
+    @Override
+    public void toApp(Message message, SessionID sessionId) {
+
+    }
+
+    @Override
+    public void fromApp(Message message, SessionID sessionId)
+            throws FieldNotFound, IncorrectTagValue, UnsupportedMessageType {
+        crack(message, sessionId);
+    }
+
+    @Override
+    public void onMessage(ExecutionReport message, SessionID sessionID) {
+    }
+}