Issue 424 - fhir-model perf tweaks and benchmarking

fhir-benchmark/pom.xml

@@ -11,6 +11,13 @@
     <name>FHIR Benchmark</name>
 
     <dependencies>
+        <!-- <dependency>
+            <groupId>com.sample</groupId>
+            <artifactId>sample</artifactId>
+            <version>1.0</version>
+            <scope>system</scope>
+            <systemPath>${project.basedir}/src/main/resources/files.jar</systemPath>
+        </dependency> -->
         <dependency>
             <groupId>org.openjdk.jmh</groupId>
             <artifactId>jmh-core</artifactId>
@@ -133,6 +140,15 @@
                     <target>${javac.target}</target>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifestFile>src/main/resources/META-INF/MANIFEST.MF</manifestFile>
+                    </archive>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRGeneratorBenchmark.java

@@ -6,13 +6,12 @@
 
 package com.ibm.fhir.benchmark;
 
-import static com.ibm.fhir.benchmark.runner.FHIRBenchmarkRunner.PROPERTY_EXAMPLE_NAME;
-
 import java.io.StringReader;
 import java.io.Writer;
 
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
@@ -27,52 +26,63 @@
 import ca.uhn.fhir.context.FhirContext;
 
 public class FHIRGeneratorBenchmark {
+    static final Writer NOP_WRITER = BenchmarkUtil.createNOPWriter();
+
+    @State(Scope.Thread)
+    public static class FHIRGenerators {
+        FHIRGenerator jsonGenerator = FHIRGenerator.generator(Format.JSON);
+        FHIRGenerator xmlGenerator = FHIRGenerator.generator(Format.XML);
+    }
+
     @State(Scope.Benchmark)
     public static class FHIRGeneratorState {
-        public static final Writer NOP_WRITER = BenchmarkUtil.createNOPWriter();
-        public static final String SPEC_EXAMPLE_NAME = System.getProperty(PROPERTY_EXAMPLE_NAME);
-        public static final String JSON_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.JSON, SPEC_EXAMPLE_NAME);
-        public static final String XML_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.XML, SPEC_EXAMPLE_NAME);
+        FhirContext context;
+        Resource resource;
+        IBaseResource baseResource;
 
-        public FhirContext context;
-        public FHIRGenerator jsonGenerator;
-        public FHIRGenerator xmlGenerator;
-        public Resource resource;
-        public IBaseResource baseResource;
+        // JMH will inject the value into the annotated field before any Setup method is called.
+        @Param({"valuesets"})
+        public String exampleName;
 
         @Setup
-        public void setUp() throws Exception {            
-            context = FhirContext.forR4();
-            jsonGenerator = FHIRGenerator.generator(Format.JSON);
-            xmlGenerator = FHIRGenerator.generator(Format.XML);
+        public void setUp() throws Exception {
+            if (exampleName == null) {
+                System.err.println("exampleName is null; if you're in Eclipse then make sure annotation processing is on and you've ran 'mvn clean package'.");
+                System.exit(1);
+            }
+
+            // us
+            String JSON_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.JSON, exampleName);
             resource = FHIRParser.parser(Format.JSON).parse(new StringReader(JSON_SPEC_EXAMPLE));
+
+            // HAPI
+            context = FhirContext.forR4();
             baseResource = context.newJsonParser().parseResource(new StringReader(JSON_SPEC_EXAMPLE));
         }
     }
 
     @Benchmark
-    public void benchmarkJsonGenerator(FHIRGeneratorState state) throws Exception {
-        state.jsonGenerator.generate(state.resource, FHIRGeneratorState.NOP_WRITER);
+    public void benchmarkJsonGenerator(FHIRGenerators generators, FHIRGeneratorState state) throws Exception {
+        generators.jsonGenerator.generate(state.resource, NOP_WRITER);
     }
 
     @Benchmark
-    public void benchmarkXMLGenerator(FHIRGeneratorState state) throws Exception {
-        state.xmlGenerator.generate(state.resource, FHIRGeneratorState.NOP_WRITER);
+    public void benchmarkXMLGenerator(FHIRGenerators generators, FHIRGeneratorState state) throws Exception {
+        generators.xmlGenerator.generate(state.resource, NOP_WRITER);
     }
 
     @Benchmark
     public void benchmarkHAPIJsonGenerator(FHIRGeneratorState state) throws Exception {
-        state.context.newJsonParser().encodeResourceToWriter(state.baseResource, FHIRGeneratorState.NOP_WRITER);
+        state.context.newJsonParser().encodeResourceToWriter(state.baseResource, NOP_WRITER);
     }
 
     @Benchmark
     public void benchmarkHAPIXMLGenerator(FHIRGeneratorState state) throws Exception {
-        state.context.newXmlParser().encodeResourceToWriter(state.baseResource, FHIRGeneratorState.NOP_WRITER);
+        state.context.newXmlParser().encodeResourceToWriter(state.baseResource, NOP_WRITER);
     }
 
     public static void main(String[] args) throws Exception {
         new FHIRBenchmarkRunner(FHIRGeneratorBenchmark.class)
-                .property(PROPERTY_EXAMPLE_NAME, BenchmarkUtil.getRandomSpecExampleName())
-                .run();
+                .run(BenchmarkUtil.getRandomSpecExampleName());
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRParserBenchmark.java

@@ -6,12 +6,11 @@
 
 package com.ibm.fhir.benchmark;
 
-import static com.ibm.fhir.benchmark.runner.FHIRBenchmarkRunner.PROPERTY_EXAMPLE_NAME;
-
+import java.io.IOException;
 import java.io.StringReader;
-import java.io.Writer;
 
 import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
@@ -20,54 +19,65 @@
 import com.ibm.fhir.benchmark.util.BenchmarkUtil;
 import com.ibm.fhir.model.format.Format;
 import com.ibm.fhir.model.parser.FHIRParser;
+import com.ibm.fhir.model.resource.Resource;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.parser.StrictErrorHandler;
 
+
 public class FHIRParserBenchmark {
+    @State(Scope.Thread)
+    public static class FHIRParsers {
+        FHIRParser jsonParser = FHIRParser.parser(Format.JSON);
+        FHIRParser xmlParser = FHIRParser.parser(Format.XML);
+    }
+
     @State(Scope.Benchmark)
     public static class FHIRParserState {
-        public static final Writer NOP_WRITER = BenchmarkUtil.createNOPWriter();
-        public static final String SPEC_EXAMPLE_NAME = System.getProperty(PROPERTY_EXAMPLE_NAME);
-        public static final String JSON_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.JSON, SPEC_EXAMPLE_NAME);
-        public static final String XML_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.XML, SPEC_EXAMPLE_NAME);
+        FhirContext context;
+        String JSON_SPEC_EXAMPLE;
+        String XML_SPEC_EXAMPLE;
 
-        public FhirContext context;
-        public FHIRParser jsonParser;
-        public FHIRParser xmlParser;
+        // JMH will inject the value into the annotated field before any Setup method is called.
+        @Param({"valuesets"})
+        public String exampleName;
 
         @Setup
-        public void setUp() {            
+        public void setUp() throws IOException {
+            if (exampleName == null) {
+                System.err.println("exampleName is null; if you're in Eclipse then make sure annotation processing is on and you've ran 'mvn clean package'.");
+                System.exit(1);
+            }
+            System.out.println("Setting up for example " + exampleName);
             context = FhirContext.forR4();
             context.setParserErrorHandler(new StrictErrorHandler());
-            jsonParser = FHIRParser.parser(Format.JSON);
-            xmlParser = FHIRParser.parser(Format.XML);
+            JSON_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.JSON, exampleName);
+            XML_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.XML, exampleName);
         }
     }
 
     @Benchmark
-    public void benchmarkJsonParser(FHIRParserState state) throws Exception {
-        state.jsonParser.parse(new StringReader(FHIRParserState.JSON_SPEC_EXAMPLE));
+    public Resource benchmarkJsonParser(FHIRParsers parsers, FHIRParserState state) throws Exception {
+        return parsers.jsonParser.parse(new StringReader(state.JSON_SPEC_EXAMPLE));
     }
 
     @Benchmark
-    public void benchmarkXMLParser(FHIRParserState state) throws Exception {
-        state.xmlParser.parse(new StringReader(FHIRParserState.XML_SPEC_EXAMPLE));
+    public Resource benchmarkXMLParser(FHIRParsers parsers, FHIRParserState state) throws Exception {
+        return parsers.xmlParser.parse(new StringReader(state.XML_SPEC_EXAMPLE));
     }
 
     @Benchmark
     public void benchmarkHAPIJsonParser(FHIRParserState state) throws Exception {
-        state.context.newJsonParser().parseResource(new StringReader(FHIRParserState.JSON_SPEC_EXAMPLE));
+        state.context.newJsonParser().parseResource(new StringReader(state.JSON_SPEC_EXAMPLE));
     }
 
     @Benchmark
     public void benchmarkHAPIXMLParser(FHIRParserState state) throws Exception {
-        state.context.newXmlParser().parseResource(new StringReader(FHIRParserState.XML_SPEC_EXAMPLE));
+        state.context.newXmlParser().parseResource(new StringReader(state.XML_SPEC_EXAMPLE));
     }
 
     public static void main(String[] args) throws Exception {
-        new FHIRBenchmarkRunner(FHIRParserBenchmark.class)
-                .property(PROPERTY_EXAMPLE_NAME, BenchmarkUtil.getRandomSpecExampleName())
-                .run();
+//        new FHIRBenchmarkRunner(FHIRParserBenchmark.class).runAll();
+        new FHIRBenchmarkRunner(FHIRParserBenchmark.class).run();
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRParserValidatorGeneratorBenchmark.java

@@ -91,7 +91,6 @@ public void benchmarkHAPIXMLParserValidatorGenerator(FHIRParserValidatorGenerato
 
     public static void main(String[] args) throws Exception {
         new FHIRBenchmarkRunner(FHIRParserValidatorGeneratorBenchmark.class)
-                .property(PROPERTY_EXAMPLE_NAME, BenchmarkUtil.getRandomSpecExampleName())
                 .run();
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRPathEvaluatorBenchmark.java

@@ -91,6 +91,6 @@ public static void main(String[] args) throws Exception {
         new FHIRBenchmarkRunner(FHIRPathEvaluatorBenchmark.class)
                 .property(PROPERTY_EXAMPLE_NAME, EXAMPLE_NAME)
                 .property(PROPERTY_EXPRESSION, EXPRESSION)
-                .run();
+                .run(BenchmarkUtil.getRandomSpecExampleName());
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRValidatorBenchmark.java

@@ -63,7 +63,6 @@ public void benchmarkHAPIValidator(FHIRValidatorState state) throws Exception {
 
     public static void main(String[] args) throws Exception {
         new FHIRBenchmarkRunner(FHIRValidatorBenchmark.class)
-                .property(PROPERTY_EXAMPLE_NAME, BenchmarkUtil.getRandomSpecExampleName())
-                .run();
+                .run(BenchmarkUtil.getRandomSpecExampleName());
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRValueSetBenchmarks.java

@@ -0,0 +1,143 @@
+/*
+ * (C) Copyright IBM Corp. 2019
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.ibm.fhir.benchmark;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.ObjectInputStream;
+import java.io.Reader;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import com.ibm.fhir.examples.ExamplesUtil;
+import com.ibm.fhir.model.format.Format;
+import com.ibm.fhir.model.parser.FHIRParser;
+import com.ibm.fhir.model.resource.ValueSet;
+import com.ibm.fhir.model.resource.ValueSet.Expansion.Contains;
+import com.ibm.fhir.model.type.Code;
+import com.ibm.fhir.model.type.DateTime;
+import com.ibm.fhir.model.type.Uri;
+import com.ibm.fhir.model.type.code.PublicationStatus;
+
+public class FHIRValueSetBenchmarks {
+    @Benchmark
+    public ValueSet readJsonResource() throws Exception {
+        Reader reader = ExamplesUtil.reader("json/ibm/valueset/ValueSet-large.json");
+        return FHIRParser.parser(Format.JSON).parse(reader);
+    }
+
+    @Benchmark
+    public ValueSet readXmlResource() throws Exception {
+        Reader reader = ExamplesUtil.reader("xml/ibm/valueset/ValueSet-large.xml");
+        return FHIRParser.parser(Format.XML).parse(reader);
+    }
+
+    @Benchmark
+    public ValueSet readTxtFile() throws Exception {
+        final ValueSet.Builder vsBuilder = ValueSet.builder().status(PublicationStatus.DRAFT);
+        final ValueSet.Expansion.Builder expansionBuilder = ValueSet.Expansion.builder().timestamp(DateTime.now());
+        final ValueSet.Expansion.Contains.Builder template = ValueSet.Expansion.Contains.builder();
+
+        InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream("ValueSet-large.txt");
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+            reader.lines()
+            .forEach(line -> {
+                String[] concept = line.split("|");
+                expansionBuilder.contains(template
+                        .system(Uri.of(concept[0]))
+                        .code(Code.of(concept[1]))
+                        .build()
+                        );
+            });
+        }
+        return vsBuilder.expansion(expansionBuilder.build()).build();
+    }
+
+    @SuppressWarnings("unchecked")
+    @Benchmark
+    public Set<String> readSerializedSet() throws Exception {
+        InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream("ValueSet-large-HashSet.ser");
+        try (ObjectInputStream is = new ObjectInputStream(in)) {
+            return (Set<String>) is.readObject();
+        }
+    }
+
+    @State(Scope.Benchmark)
+    public static class FHIRValueSetState {
+        ValueSet valueSet;
+        Contains concept;
+
+        @Setup
+        public void setUp() throws Exception {
+            Reader reader = ExamplesUtil.reader("json/ibm/valueset/ValueSet-large.json");
+            valueSet = FHIRParser.parser(Format.JSON).parse(reader);
+
+            List<Contains> concepts = valueSet.getExpansion().getContains();
+            // naive attempt to force the worst case
+            concept = concepts.get(concepts.size() - 1);
+        }
+    }
+
+    @Benchmark
+    public Set<String> buildSet(FHIRValueSetState state) throws Exception {
+        final Set<String> set = new HashSet<>();
+        state.valueSet.getExpansion().getContains().stream()
+                .forEach(concept -> set.add(concept.getSystem().getValue() + "|" + concept.getCode().getValue()));
+        return set;
+    }
+
+    @Benchmark
+    public boolean lookupInList(FHIRValueSetState state) throws Exception {
+        for (ValueSet.Expansion.Contains concept : state.valueSet.getExpansion().getContains()) {
+            if (state.concept.getCode().equals(concept.getCode()) && state.concept.getSystem().equals(concept.getSystem())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @State(Scope.Benchmark)
+    public static class FHIRHashSetState {
+        Set<String> set;
+
+        @SuppressWarnings("unchecked")
+        @Setup
+        public void setUp() throws Exception {
+            InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream("ValueSet-large-HashSet.ser");
+            try (ObjectInputStream is = new ObjectInputStream(in)) {
+                set = (Set<String>) is.readObject();
+            }
+        }
+    }
+
+    @Benchmark
+    public boolean lookupInSet(FHIRValueSetState vsState, FHIRHashSetState state) throws Exception {
+        return state.set.contains(vsState.concept.getSystem().getValue() + "|" + vsState.concept.getCode().getValue());
+    }
+
+    public static void main(String[] args) throws Exception {
+//        new FHIRBenchmarkRunner(FHIRValueSetBenchmark.class).run(BenchmarkUtil.getRandomSpecExampleName());
+        Options opt = new OptionsBuilder()
+                .include(FHIRValueSetBenchmarks.class.getSimpleName())
+                .warmupIterations(3)
+                .measurementIterations(2)
+                .forks(1)
+                .build();
+
+        new Runner(opt).run();
+    }
+}