From 9d63a88d8c052b7abdcf14f9c05ba36d9b25f992 Mon Sep 17 00:00:00 2001 From: Valentin Kulichenko Date: Fri, 6 Mar 2015 15:10:31 -0800 Subject: [PATCH] IGNITE-141 - Marshallers refactoring --- .../ignite/internal/ClassesGenerator.java | 60 +++++++++++-------- .../internal/MarshallerContextAdapter.java | 47 ++++++++++----- .../internal/MarshallerContextImpl.java | 4 +- .../GridClientOptimizedMarshaller.java | 3 +- .../ignite/internal/util/IgniteUtils.java | 30 ---------- .../optimized/OptimizedClassDescriptor.java | 2 +- .../META-INF}/classnames-jdk.properties | 0 .../META-INF}/classnames.properties | 0 .../marshaller/MarshallerContextTestImpl.java | 41 ++++--------- .../config/benchmark-multicast.properties | 24 +------- 10 files changed, 83 insertions(+), 128 deletions(-) rename modules/core/src/main/{java/org/apache/ignite/internal => resources/META-INF}/classnames-jdk.properties (100%) rename modules/core/src/main/{java/org/apache/ignite/internal => resources/META-INF}/classnames.properties (100%) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/ClassesGenerator.java b/modules/core/src/main/java/org/apache/ignite/internal/ClassesGenerator.java index 4d06dea6c7f76..6a03f0aae4fa8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/ClassesGenerator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/ClassesGenerator.java @@ -30,10 +30,10 @@ */ public class ClassesGenerator { /** */ - private static final String DFLT_BASE_PATH = U.getIgniteHome() + "/modules/core/src/main/java"; + private static final String FILE_PATH = "META-INF/classnames.properties"; /** */ - private static final String FILE_PATH = "org/apache/ignite/internal/classnames.properties"; + private static final String DFLT_BASE_PATH = U.getIgniteHome() + "/modules/core/src/main/resources"; /** */ private static final String HEADER = @@ -55,7 +55,7 @@ public class ClassesGenerator { "#"; /** */ - private static final String[] INCLUDED_PACKAGES = { + private static final String[] PACKAGES = { "org.apache.ignite", "org.jdk8.backport", "org.pcollections", @@ -69,26 +69,9 @@ public class ClassesGenerator { public static void main(String[] args) throws Exception { String basePath = args.length > 0 ? args[0] : DFLT_BASE_PATH; - File file = new File(basePath, FILE_PATH); + ClassesGenerator gen = new ClassesGenerator(basePath, HEADER, PACKAGES); - ClassesGenerator gen = new ClassesGenerator(); - - write(file, gen.generate()); - } - - /** - * @param file File. - * @param classes Classes. - * @throws Exception In case of error. - */ - private static void write(File file, Collection classes) throws Exception { - PrintStream out = new PrintStream(file); - - out.println(HEADER); - out.println(); - - for (Class cls : classes) - out.println(cls.getName()); + gen.generate(); } /** */ @@ -104,11 +87,30 @@ private static void write(File file, Collection classes) throws Exception /** */ private final Collection errs = new ArrayList<>(); + /** */ + private final String basePath; + + /** */ + private final String header; + + /** */ + private final String[] packages; + + /** + * @param basePath Base file path. + * @param header Header. + * @param packages Included packages. + */ + public ClassesGenerator(String basePath, String header, String[] packages) { + this.basePath = basePath; + this.header = header; + this.packages = packages; + } + /** * @throws Exception In case of error. - * @return Classes. */ - private Collection generate() throws Exception { + public void generate() throws Exception { System.out.println("Generating classnames.properties..."); for (URL url : ldr.getURLs()) @@ -123,7 +125,13 @@ private Collection generate() throws Exception { throw new Exception(sb.toString().trim()); } - return classes; + PrintStream out = new PrintStream(new File(basePath, FILE_PATH)); + + out.println(header); + out.println(); + + for (Class cls : classes) + out.println(cls.getName()); } /** @@ -183,7 +191,7 @@ private void processClassFile(String path, int prefixLen) boolean included = false; - for (String pkg : INCLUDED_PACKAGES) { + for (String pkg : packages) { if (clsName.startsWith(pkg)) { included = true; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java index 1fc6c5aecbca6..5e71a171c1ec5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextAdapter.java @@ -21,6 +21,8 @@ import org.jdk8.backport.*; import java.io.*; +import java.net.*; +import java.util.*; import java.util.concurrent.*; /** @@ -28,10 +30,10 @@ */ public abstract class MarshallerContextAdapter implements MarshallerContext { /** */ - private static final String[] CLS_NAMES_FILES = { - "org/apache/ignite/internal/classnames.properties", - "org/apache/ignite/internal/classnames-jdk.properties" - }; + private static final String CLS_NAMES_FILE = "META-INF/classnames.properties"; + + /** */ + private static final String JDK_CLS_NAMES_FILE = "META-INF/classnames-jdk.properties"; /** */ protected final ConcurrentMap clsNameById = new ConcurrentHashMap8<>(); @@ -43,23 +45,36 @@ public MarshallerContextAdapter() { try { ClassLoader ldr = getClass().getClassLoader(); - for (String file : CLS_NAMES_FILES) { - BufferedReader rdr = new BufferedReader(new InputStreamReader(ldr.getResourceAsStream(file))); - - String line; - - while ((line = rdr.readLine()) != null) { - if (line.isEmpty() || line.startsWith("#")) - continue; + Enumeration urls = ldr.getResources(CLS_NAMES_FILE); - String clsName = line.trim(); + while (urls.hasMoreElements()) + processResource(urls.nextElement()); - clsNameById.put(clsName.hashCode(), clsName); - } - } + processResource(ldr.getResource(JDK_CLS_NAMES_FILE)); } catch (IOException e) { throw new IllegalStateException("Failed to initialize marshaller context.", e); } } + + /** + * @param url Resource URL. + * @throws IOException In case of error. + */ + private void processResource(URL url) throws IOException { + try (InputStream in = url.openStream()) { + BufferedReader rdr = new BufferedReader(new InputStreamReader(in)); + + String line; + + while ((line = rdr.readLine()) != null) { + if (line.isEmpty() || line.startsWith("#")) + continue; + + String clsName = line.trim(); + + clsNameById.put(clsName.hashCode(), clsName); + } + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java index 4a254fb7bf5b9..5c472355459b9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java @@ -47,6 +47,8 @@ public void onMarshallerCacheReady(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void registerClass(int id, Class cls) { if (!clsNameById.containsKey(id)) { + U.debug("REG: " + cls.getName()); + try { if (cache == null) U.awaitQuiet(latch); @@ -89,6 +91,6 @@ public void onMarshallerCacheReady(GridKernalContext ctx) { } } - return U.forName(clsName, ldr); + return Class.forName(clsName, false, ldr); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/marshaller/optimized/GridClientOptimizedMarshaller.java b/modules/core/src/main/java/org/apache/ignite/internal/client/marshaller/optimized/GridClientOptimizedMarshaller.java index e63027786339b..7ca47d8436bf9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/client/marshaller/optimized/GridClientOptimizedMarshaller.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/client/marshaller/optimized/GridClientOptimizedMarshaller.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.*; import org.apache.ignite.internal.client.marshaller.*; import org.apache.ignite.internal.processors.rest.client.message.*; -import org.apache.ignite.internal.util.typedef.internal.*; import org.apache.ignite.marshaller.optimized.*; import java.io.*; @@ -112,7 +111,7 @@ private static class ClientMarshallerContext extends MarshallerContextAdapter { assert clsName != null : id; - return U.forName(clsName, ldr); + return Class.forName(clsName, false, ldr); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 50a186238fd24..d6e33399ffa69 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -266,9 +266,6 @@ public abstract class IgniteUtils { /** */ static volatile long curTimeMillis = System.currentTimeMillis(); - /** Primitive class map. */ - private static final Map> primitiveMap = new HashMap<>(16, .5f); - /** Boxed class map. */ private static final Map, Class> boxedClsMap = new HashMap<>(16, .5f); @@ -398,15 +395,6 @@ else if (archStr.contains("sparc")) IgniteUtils.javaRtName = javaRtName; IgniteUtils.javaRtVer = javaRtVer; - primitiveMap.put("byte", byte.class); - primitiveMap.put("short", short.class); - primitiveMap.put("int", int.class); - primitiveMap.put("long", long.class); - primitiveMap.put("float", float.class); - primitiveMap.put("double", double.class); - primitiveMap.put("char", char.class); - primitiveMap.put("boolean", boolean.class); - boxedClsMap.put(byte.class, Byte.class); boxedClsMap.put(short.class, Short.class); boxedClsMap.put(int.class, Integer.class); @@ -7897,24 +7885,6 @@ public static int safeAbs(int i) { return boxed != null ? boxed : cls; } - /** - * Gets class for provided name. Accepts primitive types names. - * - * @param clsName Class name. - * @param ldr Class loader. - * @return Class. - * @throws ClassNotFoundException If class not found. - */ - public static Class forName(@Nullable String clsName, @Nullable ClassLoader ldr) - throws ClassNotFoundException { - if (clsName == null) - return null; - - Class cls = primitiveMap.get(clsName); - - return cls != null ? cls : Class.forName(clsName, true, ldr); - } - /** * Applies a supplemental hash function to a given hashCode, which * defends against poor quality hash functions. This is critical diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java index 74a491f387b46..5ce69562eeb19 100644 --- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java +++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java @@ -877,7 +877,7 @@ Object read(OptimizedObjectInputStream in) throws ClassNotFoundException, IOExce return in.readBooleanArray(); case TYPE_OBJ_ARR: - return in.readArray(U.forName(in.readUTF(), in.classLoader())); + return in.readArray(Class.forName(in.readUTF(), false, in.classLoader())); case TYPE_STR: return in.readString(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classnames-jdk.properties b/modules/core/src/main/resources/META-INF/classnames-jdk.properties similarity index 100% rename from modules/core/src/main/java/org/apache/ignite/internal/classnames-jdk.properties rename to modules/core/src/main/resources/META-INF/classnames-jdk.properties diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties similarity index 100% rename from modules/core/src/main/java/org/apache/ignite/internal/classnames.properties rename to modules/core/src/main/resources/META-INF/classnames.properties diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextTestImpl.java b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextTestImpl.java index abff9b93c0901..21228326f7a4f 100644 --- a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextTestImpl.java +++ b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextTestImpl.java @@ -17,52 +17,33 @@ package org.apache.ignite.marshaller; +import org.apache.ignite.internal.*; import org.jdk8.backport.*; -import java.io.*; +import java.util.*; import java.util.concurrent.*; /** * Test marshaller context. */ -public class MarshallerContextTestImpl implements MarshallerContext { - /** */ - private static final String[] CLS_NAMES_FILES = { - "org/apache/ignite/internal/classnames.properties", - "org/apache/ignite/internal/classnames-jdk.properties" - }; - +public class MarshallerContextTestImpl extends MarshallerContextAdapter { /** */ private final ConcurrentMap map = new ConcurrentHashMap8<>(); /** */ public MarshallerContextTestImpl() { - try { - ClassLoader ldr = getClass().getClassLoader(); - - for (String file : CLS_NAMES_FILES) { - BufferedReader rdr = new BufferedReader(new InputStreamReader(ldr.getResourceAsStream(file))); - - String line; + super(); - while ((line = rdr.readLine()) != null) { - if (line.isEmpty() || line.startsWith("#")) - continue; + ClassLoader ldr = getClass().getClassLoader(); - try { - Class cls = Class.forName(line.trim(), false, ldr); - - map.put(cls.getName().hashCode(), cls); - } - catch (ClassNotFoundException | NoClassDefFoundError ignored) { - // No-op. - } - } + for (Map.Entry e : clsNameById.entrySet()) { + try { + map.put(e.getKey(), Class.forName(e.getValue(), true, ldr)); + } + catch (ClassNotFoundException | NoClassDefFoundError ignored) { + // No-op. } - } - catch (IOException e) { - throw new IllegalStateException("Failed to initialize marshaller context.", e); } } diff --git a/modules/yardstick/config/benchmark-multicast.properties b/modules/yardstick/config/benchmark-multicast.properties index f0fe47f770a6d..9d7087b3a756d 100644 --- a/modules/yardstick/config/benchmark-multicast.properties +++ b/modules/yardstick/config/benchmark-multicast.properties @@ -36,7 +36,7 @@ JVM_OPTS=${JVM_OPTS}" \ -XX:CMSInitiatingOccupancyFraction=60 \ " #Ignite version -ver="RELEASE-" +ver="141-" # List of default probes. # Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux). @@ -65,25 +65,5 @@ nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} # Run configuration which contains all benchmarks. # Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute). CONFIGS="\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxBenchmark -sn IgniteNode -ds ${ver}tx-put-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-put-get-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-val-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-1-backup\ +-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup\ "