#23: Implemented parallel layout of maps keys and values, many fixes,…

… refactorings and improvements to JPSG and lib/impl-generator by the way
leventov · Jul 6, 2014 · c219bfc · c219bfc
1 parent 96d6da8
commit c219bfc
Show file tree

Hide file tree

Showing 74 changed files with 2,797 additions and 1,871 deletions.
diff --git a/benchmarks/research/src/main/javaTemplates/net/openhft/collect/impl/ParallelVsSeparate.java b/benchmarks/research/src/main/javaTemplates/net/openhft/collect/impl/ParallelVsSeparate.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.collect.impl;
+
+import net.openhft.bench.DimensionedJmh;
+import net.openhft.collect.*;
+import net.openhft.collect.map.*;
+import net.openhft.collect.map.hash.*;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.CommandLineOptionException;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.Integer.parseInt;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Threads(1)
+@Fork(1)
+@Warmup(iterations = 5)
+@Measurement(iterations = 10)
+public class ParallelVsSeparate {
+
+    static final int BYTE_SIZE = 250;
+    static final int SMALL_CHAR_SIZE = 1000, LARGE_CHAR_SIZE = 60000;
+    static final int SMALL_SIZE = SMALL_CHAR_SIZE, LARGE_SIZE = 1000 * 1000;
+
+    static final HashConfig L_HASH_CONFIG = HashConfig.getDefault();
+    static final HashConfig Q_HASH_CONFIG = L_HASH_CONFIG.withGrowFactor(1.999);
+
+    private static int getSize(String sizeHint, String keyType) {
+        switch (keyType) {
+            case "byte":
+                return BYTE_SIZE;
+            case "char":
+                return "small".equals(sizeHint) ? SMALL_CHAR_SIZE : LARGE_CHAR_SIZE;
+            default:
+                return "small".equals(sizeHint) ? SMALL_SIZE : LARGE_SIZE;
+        }
+    }
+
+    /* with char|byte|int|long key QHash|LHash hash */
+
+    @State(Scope.Thread)
+    public static class QHashCharCharMapState {
+        Random r;
+        char[] keys;
+        CharCharMap map;
+
+        @Setup(Level.Trial)
+        public void allocate() {
+            r = ThreadLocalRandom.current();
+            int size = getSize(System.getProperty("size", "small"), char.class.getName());
+            keys = new char[size];
+            HashCharCharMapFactory factory = HashCharCharMaps.getDefaultFactory();
+            factory = factory.withConfig(factory.getConfig().withHashConfig(Q_HASH_CONFIG));
+            map = factory.newUpdatableMap(size);
+        }
+
+        @Setup(Level.Invocation)
+        public void generateKeys() {
+            for (int i = 0; i < keys.length; i++) {
+                keys[i] = (char) r.nextLong();
+            }
+        }
+
+        @TearDown(Level.Trial)
+        public void recycle() {
+            keys = null;
+            map = null;
+        }
+    }
+
+    public static class PutOpQHashCharCharMapState extends QHashCharCharMapState {
+        @Setup(Level.Invocation)
+        public void clearMap() {
+            map.clear();
+        }
+    }
+
+    public static class QueryUpdateOpQHashCharCharMapState extends QHashCharCharMapState {
+        @Setup(Level.Invocation)
+        public void fillMap() {
+            map.clear();
+            for (char key : keys) {
+                map.put(key, /* const key 1 */(char) 1/* endconst */);
+            }
+        }
+    }
+
+    @GenerateMicroBenchmark
+    public long putOp_qHash_charKey(PutOpQHashCharCharMapState state) {
+        char[] keys = state.keys;
+        CharCharMap map = state.map;
+        long dummy = 0L;
+        for (char key : keys) {
+            dummy ^= (long) map.put(key, /* const key 1 */(char) 1/* endconst */);
+        }
+        return dummy;
+    }
+
+    @GenerateMicroBenchmark
+    public long getOp_qHash_charKey(QueryUpdateOpQHashCharCharMapState state) {
+        char[] keys = state.keys;
+        CharCharMap map = state.map;
+        long dummy = 0L;
+        for (char key : keys) {
+            dummy ^= (long) map.get(key);
+        }
+        return dummy;
+    }
+
+    @GenerateMicroBenchmark
+    public long incrementValueOp_qHash_charKey(QueryUpdateOpQHashCharCharMapState state) {
+        char[] keys = state.keys;
+        CharCharMap map = state.map;
+        long dummy = 0L;
+        for (char key : keys) {
+            dummy ^= (long) map.incrementValue(key, /* const key 1 */(char) 1/* endconst */);
+        }
+        return dummy;
+    }
+
+    @GenerateMicroBenchmark
+    public long computeOp_qHash_charKey(QueryUpdateOpQHashCharCharMapState state) {
+        char[] keys = state.keys;
+        CharCharMap map = state.map;
+        long dummy = 0L;
+        for (char key : keys) {
+            dummy ^= (long) map.compute(key, (k, v) -> (char) (-v));
+        }
+        return dummy;
+    }
+
+    /* endwith */
+
+    public static void main(String[] args) throws RunnerException, CommandLineOptionException {
+        new DimensionedJmh(ParallelVsSeparate.class)
+                .addArgDim("size", "small", "large")
+                .withGetOperationsPerInvocation(options ->
+                        getSize(options.get("size"), options.get("key")))
+                .run(args);
+    }
+}
diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/BitsModifierPostProcessor.java b/jpsg/core/src/main/java/net/openhft/jpsg/BitsModifierPostProcessor.java
@@ -16,12 +16,10 @@
 
 package net.openhft.jpsg;
 
-import java.util.function.UnaryOperator;
-
 
 public final class BitsModifierPostProcessor extends PrimitiveTypeModifierPostProcessor {
 
     public BitsModifierPostProcessor() {
-        super("bits", PrimitiveType::bitsType);
+        super("bits", PrimitiveType::bitsType, dim -> true);
     }
 }
diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/BitsModifierPreProcessor.java b/jpsg/core/src/main/java/net/openhft/jpsg/BitsModifierPreProcessor.java
@@ -19,6 +19,6 @@
 public final class BitsModifierPreProcessor extends PrimitiveTypeModifierPreProcessor {
 
     public BitsModifierPreProcessor() {
-        super("bits", PrimitiveType::bitsType);
+        super("bits", PrimitiveType::bitsType, dim -> true);
     }
 }
diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/Condition.java b/jpsg/core/src/main/java/net/openhft/jpsg/Condition.java
@@ -30,7 +30,7 @@ public final class Condition {
      */
 
     private static final String POSSIBLY_NEGATED_DIMENSIONS =
-            format("(%s|!\\(%s\\))", DIMENSIONS, DIMENSIONS);
+            format("(%s|!?\\(%s\\))", DIMENSIONS, DIMENSIONS);
 
     static final String CONDITION =
             format("((%s\\s*\\|\\|\\s*)*|(%s\\s*&&\\s*)*)\\s*%s",
@@ -62,12 +62,14 @@ static Condition parse(String condition, Dimensions.Parser dimensionsParser, Con
         }
         for (String dims : allDims) {
             dims = dims.trim();
-            if (dims.startsWith("!(")) {
+            if (dims.startsWith("!")) {
                 cond.negated.add(true);
-                dims = dims.substring(2, dims.length() - 1);
+                dims = dims.substring(1);
             } else {
                 cond.negated.add(false);
             }
+            if (dims.startsWith("("))
+                dims = dims.substring(1, dims.length() - 1);
             cond.allDims.add(dimensionsParser.parseForCondition(dims, context));
         }
         return cond;

diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/ConstProcessor.java b/jpsg/core/src/main/java/net/openhft/jpsg/ConstProcessor.java
@@ -23,7 +23,7 @@
 
 public final class ConstProcessor extends TemplateProcessor {
     // after option processor because constant "(char) 0" is generated
-    public static final int PRIORITY = OptionProcessor.PRIORITY - 1;
+    public static final int PRIORITY = OptionProcessor.PRIORITY - 10;
 
     private static CheckingPattern valuePattern(String dim) {
         String prefix = "/[\\*/]\\s*const\\s+" + dim;

diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/DefinitionProcessor.java b/jpsg/core/src/main/java/net/openhft/jpsg/DefinitionProcessor.java
@@ -21,7 +21,7 @@
 
 
 public final class DefinitionProcessor extends TemplateProcessor {
-    public static final int PRIORITY = Generator.BlocksProcessor.PRIORITY + 10;
+    public static final int PRIORITY = Generator.BlocksProcessor.PRIORITY + 100;
 
     private static final String DEF_PREFIX = "/[\\*/]\\s*define";
     private static final CheckingPattern DEF_P = CheckingPattern.compile(DEF_PREFIX,

diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/Generator.java b/jpsg/core/src/main/java/net/openhft/jpsg/Generator.java
@@ -26,6 +26,7 @@
 import java.util.*;
 import java.util.concurrent.ForkJoinTask;
 import java.util.concurrent.RecursiveAction;
+import java.util.function.Predicate;
 import java.util.function.UnaryOperator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -173,9 +174,9 @@ public Generator addProcessor(TemplateProcessor processor) {
     }
 
     public Generator addPrimitiveTypeModifierProcessors(String keyword,
-            UnaryOperator<PrimitiveType> typeMapper) {
-        addProcessor(new PrimitiveTypeModifierPreProcessor(keyword, typeMapper));
-        addProcessor(new PrimitiveTypeModifierPostProcessor(keyword, typeMapper));
+            UnaryOperator<PrimitiveType> typeMapper, Predicate<String> dimFilter) {
+        addProcessor(new PrimitiveTypeModifierPreProcessor(keyword, typeMapper, dimFilter));
+        addProcessor(new PrimitiveTypeModifierPostProcessor(keyword, typeMapper, dimFilter));
         return this;
     }
 
@@ -478,7 +479,7 @@ private boolean checkContext(Context target) {
     }
 
     public final class BlocksProcessor extends TemplateProcessor {
-        public static final int PRIORITY = DEFAULT_PRIORITY + 10;
+        public static final int PRIORITY = DEFAULT_PRIORITY + 100;
 
         @Override
         protected int priority() {

diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/PrimitiveTypeModifierPostProcessor.java b/jpsg/core/src/main/java/net/openhft/jpsg/PrimitiveTypeModifierPostProcessor.java
@@ -17,19 +17,22 @@
 package net.openhft.jpsg;
 
 import java.util.Map;
+import java.util.function.Predicate;
 import java.util.function.UnaryOperator;
 
 
 public class PrimitiveTypeModifierPostProcessor extends TemplateProcessor {
-    public static final int PRIORITY = OptionProcessor.PRIORITY - 1;
+    public static final int PRIORITY = OptionProcessor.PRIORITY - 10;
 
     private final String keyword;
     private final UnaryOperator<PrimitiveType> typeMapper;
+    private final Predicate<String> dimFilter;
 
     public PrimitiveTypeModifierPostProcessor(String keyword,
-            UnaryOperator<PrimitiveType> typeMapper) {
+            UnaryOperator<PrimitiveType> typeMapper, Predicate<String> dimFilter) {
         this.keyword = keyword;
         this.typeMapper = typeMapper;
+        this.dimFilter = dimFilter;
     }
 
     @Override
@@ -41,10 +44,15 @@ protected int priority() {
     protected void process(StringBuilder sb, Context source, Context target, String template) {
         for (Map.Entry<String, Option> e : target) {
             String dim = e.getKey();
+            if (!dimFilter.test(dim))
+                continue;
             Option targetT = e.getValue();
-            if (targetT instanceof PrimitiveType) {
+            if (targetT instanceof PrimitiveType || targetT instanceof ObjectType) {
                 String kwDim = dim + "." + keyword;
-                template = typeMapper.apply((PrimitiveType) targetT).finalReplace(template, kwDim);
+                Option mapped = targetT instanceof PrimitiveType ?
+                        typeMapper.apply((PrimitiveType) targetT) :
+                        targetT; // ObjectType maps to itself
+                template = mapped.finalReplace(template, kwDim);
             }
         }
         postProcess(sb, source, target, template);

diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/PrimitiveTypeModifierPreProcessor.java b/jpsg/core/src/main/java/net/openhft/jpsg/PrimitiveTypeModifierPreProcessor.java
@@ -17,19 +17,22 @@
 package net.openhft.jpsg;
 
 import java.util.Map;
+import java.util.function.Predicate;
 import java.util.function.UnaryOperator;
 
 
 public class PrimitiveTypeModifierPreProcessor extends TemplateProcessor {
-    public static final int PRIORITY = OptionProcessor.PRIORITY + 1;
+    public static final int PRIORITY = OptionProcessor.PRIORITY + 10;
 
     private final String keyword;
     private final UnaryOperator<PrimitiveType> typeMapper;
+    private final Predicate<String> dimFilter;
 
     public PrimitiveTypeModifierPreProcessor(String keyword,
-            UnaryOperator<PrimitiveType> typeMapper) {
+            UnaryOperator<PrimitiveType> typeMapper, Predicate<String> dimFilter) {
         this.keyword = keyword;
         this.typeMapper = typeMapper;
+        this.dimFilter = dimFilter;
     }
 
     @Override
@@ -42,11 +45,14 @@ protected void process(StringBuilder sb, Context source, Context target, String
         String modifier = OptionProcessor.modifier(keyword);
         for (Map.Entry<String, Option> e : source) {
             String dim = e.getKey();
-            Option targetT = target.getOption(dim);
-            if (e.getValue() instanceof PrimitiveType && targetT instanceof PrimitiveType) {
+            if (!dimFilter.test(dim))
+                continue;
+            if (e.getValue() instanceof PrimitiveType) {
+                Option targetT = target.getOption(dim);
                 PrimitiveType sourceT = (PrimitiveType) e.getValue();
                 String kwDim = dim + "." + keyword;
-                if (typeMapper.apply((PrimitiveType) targetT) != targetT) {
+                if (targetT instanceof PrimitiveType &&
+                        typeMapper.apply((PrimitiveType) targetT) != targetT) {
                     String modP = OptionProcessor.prefixPattern(modifier, sourceT.standalone);
                     template = template.replaceAll(modP, IntermediateOption.of(kwDim).standalone);
                 }

diff --git a/jpsg/core/src/main/java/net/openhft/jpsg/RawModifierProcessor.java b/jpsg/core/src/main/java/net/openhft/jpsg/RawModifierProcessor.java
@@ -23,7 +23,7 @@ public final class RawModifierProcessor extends TemplateProcessor {
     /**
      * {@code RawModifierProcessor} should run before any {@link PrimitiveTypeModifierPreProcessor}
      */
-    public static final int PRIORITY = PrimitiveTypeModifierPreProcessor.PRIORITY + 1;
+    public static final int PRIORITY = PrimitiveTypeModifierPreProcessor.PRIORITY + 10;
 
     private static final String RAW = OptionProcessor.modifier("raw");
 

diff --git a/jpsg/gradle-plugin/src/main/java/net/openhft/jpsg/GeneratorTask.java b/jpsg/gradle-plugin/src/main/java/net/openhft/jpsg/GeneratorTask.java
@@ -22,6 +22,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.function.Predicate;
 import java.util.function.UnaryOperator;
 
 
@@ -49,6 +50,10 @@ public GeneratorTask with(String... defaultContext) {
         return this;
     }
 
+    public GeneratorTask addToDefaultContext(String... defaultContext) {
+        return with(defaultContext);
+    }
+
     public GeneratorTask addProcessor(TemplateProcessor processor) {
         g.addProcessor(processor);
         return this;
@@ -65,8 +70,8 @@ public GeneratorTask addProcessor(String processorClassName) {
     }
 
     public GeneratorTask addPrimitiveTypeModifierProcessors(String keyword,
-            UnaryOperator<PrimitiveType> typeMapper) {
-        g.addPrimitiveTypeModifierProcessors(keyword, typeMapper);
+            UnaryOperator<PrimitiveType> typeMapper, Predicate<String> dimFilter) {
+        g.addPrimitiveTypeModifierProcessors(keyword, typeMapper, dimFilter);
         return this;
     }